Lines Matching +full:parallel +full:- +full:memories
1 /* SPDX-License-Identifier: GPL-2.0
3 * Copyright 2016-2023 HabanaLabs, Ltd.
19 #include <linux/dma-direction.h>
28 #include <linux/io-64-nonatomic-lo-hi.h>
30 #include <linux/dma-buf.h>
45 * bits[63:59] - Encode mmap type
46 * bits[45:0] - mmap offset value
51 #define HL_MMAP_TYPE_SHIFT (59 - PAGE_SHIFT)
110 * enum hl_mmu_page_table_location - mmu page table location
111 * @MMU_DR_PGT: page-table is located on device DRAM.
112 * @MMU_HR_PGT: page-table is located on host memory.
113 * @MMU_NUM_PGT_LOCATIONS: number of page-table locations currently supported.
116 MMU_DR_PGT = 0, /* device-dram-resident MMU PGT */
135 #define IS_POWER_OF_2(n) (n != 0 && ((n & (n - 1)) == 0))
168 * - HL_DRV_RESET_HARD
172 * - HL_DRV_RESET_FROM_RESET_THR
173 * Set if the caller is the hard-reset thread
175 * - HL_DRV_RESET_HEARTBEAT
178 * - HL_DRV_RESET_TDR
181 * - HL_DRV_RESET_DEV_RELEASE
184 * - HL_DRV_RESET_BYPASS_REQ_TO_FW
188 * - HL_DRV_RESET_FW_FATAL_ERR
191 * - HL_DRV_RESET_DELAY
194 * - HL_DRV_RESET_FROM_WD_THR
220 #define HL_BLOCK_GLBL_SEC_SIZE (HL_BLOCK_SIZE - HL_BLOCK_GLBL_SEC_OFFS)
231 * struct iterate_module_ctx - HW module iterator
234 * @rc: return code for optional use of iterator/iterator-caller
242 * @offset: current HW module instance offset from the 1-st HW module instance
243 * in the 1-st block
259 * struct hl_gen_wait_properties - properties for generating a wait CB
279 * struct pgt_info - MMU hop page info.
280 * @node: hash linked-list node for the pgts on host (shadow pgts for device resident MMU and
307 * enum hl_pci_match_mode - pci match mode per region
317 * enum hl_fw_component - F/W components to read version through registers.
329 * enum hl_fw_types - F/W types present in the system
333 * @FW_TYPE_PREBOOT_CPU: Indicates pre-loaded CPUs are present in the system
347 * enum hl_queue_type - Supported QUEUE types.
352 * memories and/or operates the compute engines.
378 * struct hl_inbound_pci_region - inbound region descriptor
394 * struct hl_outbound_pci_region - outbound region descriptor
404 * enum queue_cb_alloc_flags - Indicates queue support for CBs that
415 * struct hl_hw_sob - H/W SOB info.
439 * struct hw_queue_properties - queue information.
461 * enum vm_type - virtual memory mapping request information.
471 * enum mmu_op_flags - mmu operation relevant information.
486 * enum hl_device_hw_state - H/W device state. use this to understand whether
500 * struct hl_mmu_properties - ASIC specific MMU address translation properties.
532 * struct hl_hints_range - hint addresses reserved va range.
542 * struct asic_fixed_properties - ASIC specific immutable properties.
546 * @cpucp_info: received various information from CPU-CP regarding the H/W, e.g.
548 * @uboot_ver: F/W U-boot version.
559 * @sram_user_base_address - SRAM physical start address for user access.
636 * 1- enabled, 0- isolated.
638 * 1- faulty cluster, 0- good cluster.
640 * 1- enabled, 0- isolated.
663 * @eq_interrupt_id: interrupt id for EQ, uses to synchronize EQ interrupts in hard-reset.
668 * @completion_mode: 0 - job based completion, 1 - cs based completion
669 * @mme_master_slave_mode: 0 - Each MME works independently, 1 - MME works
687 * @supports_compute_reset: is a reset which is not a hard-reset supported by this asic.
690 * in inference ASICs, as there is no real-world
691 * use-case of doing soft-reset in training (due
694 * @configurable_stop_on_err: is stop-on-error option configurable via debugfs.
825 * struct hl_fence - software synchronization primitive
848 * struct hl_cs_compl - command submission completion object.
855 * @type: type of the CS - signal/wait.
879 * struct hl_ts_buff - describes a timestamp buffer.
893 * struct hl_mem_mgr - describes unified memory manager for mappable memory chunks.
905 * struct hl_mmap_mem_buf_behavior - describes unified memory manager buffer behavior
924 * struct hl_mmap_mem_buf - describes a single unified memory buffer
948 * struct hl_cb - describes a Command Buffer.
1006 /* Host <-> CPU-CP shared memory size */
1010 * struct hl_sync_stream_properties -
1037 * struct hl_encaps_signals_mgr - describes sync stream encapsulated signals
1048 * struct hl_hw_queue - describes a H/W transport queue.
1083 * struct hl_cq - describes a completion queue
1112 * struct hl_ts_free_jobs - holds user interrupt ts free nodes related data
1128 * struct hl_user_interrupt - holds user interrupt information
1152 * struct timestamp_reg_free_node - holds the timestamp registration free objects node
1167 /* struct timestamp_reg_work_obj - holds the timestamp registration free objects job
1183 /* struct timestamp_reg_info - holds the timestamp registration related data.
1205 * struct hl_user_pending_interrupt - holds a context to a user thread
1223 * struct hl_eq - describes the event queue (single one per device)
1244 * struct hl_dec - describes a decoder sw instance.
1258 * enum hl_asic_type - supported ASIC types.
1260 * @ASIC_GOYA: Goya device (HL-1000).
1261 * @ASIC_GAUDI: Gaudi device (HL-2000).
1262 * @ASIC_GAUDI_SEC: Gaudi secured device (HL-2000).
1280 * enum hl_pm_mng_profile - power management profile.
1292 * enum hl_pll_frequency - PLL frequency.
1330 * struct pci_mem_region - describe memory region in a PCI bar
1348 * struct static_fw_load_mgr - static FW load manager
1351 * @kmd_msg_to_cpu_reg: register address for KDM->CPU messages
1380 * struct fw_response - FW response to LKD command
1392 * struct dynamic_fw_load_mgr - dynamic FW load manager
1410 * struct pre_fw_load_props - needed properties for pre-FW load
1431 * struct fw_image_props - properties of FW image
1443 * struct fw_load_mgr - manager FW loading process
1476 * struct engines_data - asic engines data
1488 * struct hl_asic_funcs - ASIC specific functions that are can be called from
1492 * @late_init: sets up late driver/hw state (post hw_init) - Optional.
1493 * @late_fini: tears down what was done in late_init (pre hw_fini) - Optional.
1505 * @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific
1527 * @dma_unmap_sgtable: DMA unmap scatter-gather table.
1528 * @dma_map_sgtable: DMA map scatter-gather table.
1537 * @handle_eqe: handle event queue entry (IRQ) from CPU-CP.
1543 * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with ASID-VA-size mask.
1544 * @mmu_prefetch_cache_range: pre-fetch specific MMU STLB cache lines with ASID-VA-size mask.
1545 * @send_heartbeat: send is-alive packet to CPU-CP and verify response.
1567 * @pre_schedule_cs: Perform pre-CS-scheduling operations.
1585 * @descramble_addr: Routine to de-scramble the address prior of
1595 * after every hard-reset of the device
1597 * @get_msi_info: Retrieve asic-specific MSI ID of the f/w async event
1600 * @init_firmware_preload_params: initialize pre FW-load parameters.
1759 * enum hl_va_range_type - virtual address range type.
1772 * struct hl_va_range - virtual addresses range.
1788 * struct hl_cs_counters_atomic - command submission counters
1793 * @max_cs_in_flight_drop_cnt: dropped due to maximum CS in-flight
1806 * struct hl_dmabuf_priv - a dma-buf private object.
1807 * @dmabuf: pointer to dma-buf object.
1808 * @ctx: pointer to the dma-buf owner's context.
1809 * @phys_pg_pack: pointer to physical page pack if the dma-buf was exported
1815 * if phys_pg_pack is NULL (dma-buf was exported from address).
1830 * struct hl_cs_outcome - represents a single completed CS outcome
1846 * struct hl_cs_outcome_store - represents a limited store of completed CS outcomes
1850 * @nodes_pool: a static pool of pre-allocated outcome objects
1862 * struct hl_ctx - user/kernel context.
1866 * @hr_mmu_phys_hash: if host-resident MMU is used, holds a mapping from
1867 * MMU-hop-page physical address to its host-resident
1934 * struct hl_ctx_mgr - for handling multiple contexts.
1949 * struct hl_userptr - memory mapping chunk information
1951 * @job_node: linked-list node for hanging the object on the Job's list.
1954 * @sgt: pointer to the scatter-gather table that holds the pages.
1958 * @addr: user-space virtual address of the start of the memory area.
1977 * struct hl_cs - command submission.
2057 * struct hl_cs_job - command submission job.
2064 * @userptr_list: linked-list of userptr mappings that belong to this job and
2077 * handle to a kernel-allocated CB object, false
2107 * struct hl_cs_parser - command submission parser properties.
2111 * @job_userptr_list: linked-list of userptr mappings that belong to the related
2121 * handle to a kernel-allocated CB object, false
2152 * struct hl_vm_hash_node - hash element from virtual address to virtual
2170 * struct hl_vm_hw_block_list_node - list element from user virtual address to
2176 * @mapped_size: size of the block which is mapped. May change if partial un-mappings are done.
2189 * struct hl_vm_phys_pg_pack - physical page pack.
2222 * struct hl_vm_va_block - virtual range block information.
2236 * struct hl_vm - virtual memory manager for MMU.
2258 * struct hl_debug_params - Coresight debug parameters.
2276 * struct hl_notifier_event - holds the notifier data structure
2292 * struct hl_fpriv - process information stored in FD private data.
2328 * struct hl_info_list - debugfs file ops.
2341 * struct hl_debugfs_entry - debugfs dentry wrapper.
2351 * struct hl_dbg_device_entry - ASIC specific debugfs manager.
2415 * struct hl_hw_obj_name_entry - single hw object name, member of
2468 * struct hl_mon_state_dump - represents a state dump of a single monitor
2486 * struct hl_sync_to_engine_map_entry - sync object id to engine mapping entry
2499 * struct hl_sync_to_engine_map - maps sync object id to associated engine id
2508 * struct hl_state_dump_specs_funcs - virtual functions used by the state dump
2530 * struct hl_state_dump_specs - defines ASIC known hw objects names
2566 #define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg))
2567 #define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v))
2569 hdev->asic_funcs->rreg(hdev, (reg)))
2629 __rc ? __rc : ((cond) ? 0 : -ETIMEDOUT); \
2643 * TODO: use read from PCI bar in other places in the code (SW-91406)
2657 __rc = -EINVAL; \
2659 __elem_bitmask = BIT_ULL(arr_size) - 1; \
2683 __rc ? __rc : ((__elem_bitmask == 0) ? 0 : -ETIMEDOUT); \
2706 * might need to be byte-swapped before returning value to caller.
2732 (cond) ? 0 : -ETIMEDOUT; \
2739 p->address = base; \
2740 p->size = sz; \
2757 * struct hl_device_reset_work - reset work wrapper.
2769 * struct hl_mmu_hr_pgt_priv - used for holding per-device mmu host-resident
2770 * page-table internal information.
2771 * @mmu_pgt_pool: pool of page tables used by a host-resident MMU for
2773 * @mmu_asid_hop0: per-ASID array of host-resident hop0 tables.
2781 * struct hl_mmu_dr_pgt_priv - used for holding per-device mmu device-resident
2782 * page-table internal information.
2792 * struct hl_mmu_priv - used for holding per-device mmu internal information.
2793 * @dr: information on the device-resident MMU, when exists.
2794 * @hr: information on the host-resident MMU, when exists.
2802 * struct hl_mmu_per_hop_info - A structure describing one TLB HOP and its entry
2816 * struct hl_mmu_hop_info - A structure describing the TLB hops and their
2817 * hop-entries that were created in order to translate a virtual address to a
2820 * address replaces the original virtual-address when mapped
2822 * @unscrambled_paddr: The un-scrambled physical address.
2823 * @hop_info: Array holding the per-hop information used for the translation.
2836 * struct hl_hr_mmu_funcs - Device related host resident MMU functions.
2852 * struct hl_mmu_funcs - Device related MMU functions.
2862 * @get_tlb_info: returns the list of hops and hop-entries used that were
2883 * struct hl_prefetch_work - prefetch work structure handler
2886 * @va: virtual address to pre-fetch.
2887 * @size: pre-fetch size.
2902 * parallel
2907 * struct multi_cs_completion - multi CS wait completion.
2910 * @timestamp: timestamp for the multi-CS completion
2911 * @stream_master_qid_map: bitmap of all stream masters on which the multi-CS
2924 * struct multi_cs_data - internal data for multi CS call
2931 * @completion_bitmap: bitmap of completed CSs (1- completed, otherwise 0)
2933 * @gone_cs: indication of gone CS (1- there was gone CS, otherwise 0)
2934 * @update_ts: update timestamp. 1- update the timestamp, otherwise 0.
2950 * struct hl_clk_throttle_timestamp - current/last clock throttling timestamp
2960 * struct hl_clk_throttle - keeps current/last clock throttling timestamps
2976 * struct user_mapped_block - describes a hw block allowed to be mmapped by user
2986 * struct cs_timeout_info - info of last CS timeout occurred.
2988 * @write_enable: if set writing to CS parameters in the structure is enabled. otherwise - disabled,
3001 * struct undefined_opcode_info - info about last undefined opcode error
3005 * filled with values, it means the execution was in Lower-CP.
3008 * @cb_addr_streams_len: num of streams - actual len of cb_addr_streams array.
3010 * in Upper-CP (specific stream) and equal to 4 in case
3011 * of undefined opcode in Lower-CP.
3012 * @engine_id: engine-id that the error occurred on
3014 * MAX_QMAN_STREAMS_INFO it means the error occurred on a Lower-CP.
3031 * struct page_fault_info - page fault information.
3035 * @page_fault_detected: if set as 1, then a page-fault was discovered for the
3036 * first time after the driver has finished booting-up.
3037 * Since we're looking for the page-fault's root cause,
3038 * we don't care of the others that might follow it-
3051 * struct razwi_info - RAZWI information.
3054 * first time after the driver has finished booting-up.
3056 * we don't care of the others that might follow it-
3067 * struct hw_err_info - HW error information.
3070 * first time after the driver has finished booting-up.
3071 * currently we assume that only fatal events (that require hard-reset) are
3084 * struct fw_err_info - FW error information.
3087 * first time after the driver has finished booting-up.
3088 * currently we assume that only fatal events (that require hard-reset) are
3101 * struct engine_err_info - engine error information.
3104 * first time after the driver has finished booting-up.
3115 * struct hl_error_info - holds information collected during an error.
3135 * struct hl_reset_info - holds current device reset information.
3142 * @in_compute_reset: Device is currently in reset but not in hard-reset.
3173 * struct hl_device - habanalabs device structure.
3183 * @work_heartbeat: delayed work for CPU-CP is-alive check.
3207 * @prefetch_wq: work queue for MMU pre-fetch operations.
3214 * @event_queue: event queue for IRQ from CPU-CP.
3216 * @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address.
3217 * @cpu_accessible_dma_address: Host <-> CPU-CP shared memory DMA address.
3218 * @cpu_accessible_dma_pool: Host <-> CPU-CP shared memory pool.
3221 * @send_cpu_message_lock: enforces only one message in Host <-> CPU-CP queue.
3250 * @mmu_priv: device-specific MMU data.
3251 * @mmu_func: device-related MMU functions.
3256 * @multi_cs_completion: array of multi-CS completion.
3265 * @fw_sw_sub_minor_ver: the sub-minor of current loaded preboot SW version.
3270 * value is saved so in case of hard-reset, the driver will restore
3271 * this value and update the F/W after the re-initialization
3290 * drams are binned-out
3292 * tpc engines are binned-out
3293 * @dmabuf_export_cnt: number of dma-buf exporting.
3299 * indicates which decoder engines are binned-out
3301 * indicates which edma engines are binned-out
3304 * which indicates which rotator engines are binned-out(Gaudi3 and above).
3307 * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit
3317 * such as context-switch, context close, page free, etc.
3356 * @hard_reset_on_fw_events: Whether to do device hard-reset when a fatal event is received from
3512 /* Parameters for bring-up to be upstreamed */
3526 ((hdev)->pdev ? dev_name(&(hdev)->pdev->dev) : "NA-DEVICE")
3529 * struct hl_cs_encaps_sig_handle - encapsulated signals handle structure
3554 * struct hl_info_fw_err_info - firmware error information structure
3572 * typedef hl_ioctl_t - typedef for ioctl function in the driver
3582 * struct hl_ioctl_desc - describes an IOCTL entry of the driver.
3593 if (hdev->fw_sw_major_ver < fw_sw_major) in hl_is_fw_sw_ver_below()
3595 if (hdev->fw_sw_major_ver > fw_sw_major) in hl_is_fw_sw_ver_below()
3597 if (hdev->fw_sw_minor_ver < fw_sw_minor) in hl_is_fw_sw_ver_below()
3605 return (hdev->fw_sw_major_ver > fw_sw_major || in hl_is_fw_sw_ver_equal_or_greater()
3606 (hdev->fw_sw_major_ver == fw_sw_major && in hl_is_fw_sw_ver_equal_or_greater()
3607 hdev->fw_sw_minor_ver >= fw_sw_minor)); in hl_is_fw_sw_ver_equal_or_greater()
3615 * hl_get_sg_info() - get number of pages and the DMA address from SG list.
3627 return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) + in hl_get_sg_info()
3628 (PAGE_SIZE - 1)) >> PAGE_SHIFT; in hl_get_sg_info()
3632 * hl_mem_area_inside_range() - Checks whether address+size are inside a range.
3659 * hl_mem_area_crosses_range() - Checks whether address+size crossing a range.
3671 u64 end_address = address + size - 1; in hl_mem_area_crosses_range()
3727 #define hl_pi_2_offset(pi) ((pi) & (HL_QUEUE_LENGTH - 1))