22 #include "ipa.h"
41  * This driver supports the Qualcomm IP Accelerator (IPA), which is a
42  * networking component found in many Qualcomm SoCs.  The IPA is connected
46  * The IPA is the conduit between the AP and the modem that carries network
50  * The IPA provides protocol checksum calculation, offloading this work
51  * from the AP.  The IPA offers additional functionality, including routing,
57  * There are two distinct layers that implement the IPA hardware, and this
59  * interface (GSI) is an integral component of the IPA, providing a
60  * well-defined communication layer between the AP subsystem and the IPA
62  * between the AP and the IPA.
64  * The IPA layer uses GSI channels to implement its "endpoints".  And while
65  * a GSI channel carries data between the AP and the IPA, a pair of IPA
77 /* IPA v5.5+ does not specify Qtime timestamp config for DPL */
103  * ipa_setup() - Set up IPA hardware
104  * @ipa:	IPA pointer
113 int ipa_setup(struct ipa *ipa)  in ipa_setup()  argument
117 	struct device *dev = &ipa->pdev->dev;  in ipa_setup()
120 	ret = gsi_setup(&ipa->gsi);  in ipa_setup()
124 	ret = ipa_power_setup(ipa);  in ipa_setup()
128 	ipa_endpoint_setup(ipa);  in ipa_setup()
133 	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];  in ipa_setup()
138 	ret = ipa_mem_setup(ipa);	/* No matching teardown required */  in ipa_setup()
142 	ret = ipa_table_setup(ipa);	/* No matching teardown required */  in ipa_setup()
149 	exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX];  in ipa_setup()
154 	ipa_endpoint_default_route_set(ipa, exception_endpoint->endpoint_id);  in ipa_setup()
157 	ret = ipa_qmi_setup(ipa);  in ipa_setup()
161 	ipa->setup_complete = true;  in ipa_setup()
163 	dev_info(dev, "IPA driver setup completed successfully\n");  in ipa_setup()
168 	ipa_endpoint_default_route_clear(ipa);  in ipa_setup()
173 	ipa_endpoint_teardown(ipa);  in ipa_setup()
174 	ipa_power_teardown(ipa);  in ipa_setup()
176 	gsi_teardown(&ipa->gsi);  in ipa_setup()
183  * @ipa:	IPA pointer
185 static void ipa_teardown(struct ipa *ipa)  in ipa_teardown()  argument
191 	ipa->setup_complete = false;  in ipa_teardown()
193 	ipa_qmi_teardown(ipa);  in ipa_teardown()
194 	ipa_endpoint_default_route_clear(ipa);  in ipa_teardown()
195 	exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX];  in ipa_teardown()
197 	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];  in ipa_teardown()
199 	ipa_endpoint_teardown(ipa);  in ipa_teardown()
200 	ipa_power_teardown(ipa);  in ipa_teardown()
201 	gsi_teardown(&ipa->gsi);  in ipa_teardown()
205 ipa_hardware_config_bcr(struct ipa *ipa, const struct ipa_data *data)  in ipa_hardware_config_bcr()  argument
210 	/* IPA v4.5+ has no backward compatibility register */  in ipa_hardware_config_bcr()
211 	if (ipa->version >= IPA_VERSION_4_5)  in ipa_hardware_config_bcr()
214 	reg = ipa_reg(ipa, IPA_BCR);  in ipa_hardware_config_bcr()
216 	iowrite32(val, ipa->reg_virt + reg_offset(reg));  in ipa_hardware_config_bcr()
219 static void ipa_hardware_config_tx(struct ipa *ipa)  in ipa_hardware_config_tx()  argument
221 	enum ipa_version version = ipa->version;  in ipa_hardware_config_tx()
230 	reg = ipa_reg(ipa, IPA_TX_CFG);  in ipa_hardware_config_tx()
233 	val = ioread32(ipa->reg_virt + offset);  in ipa_hardware_config_tx()
237 	iowrite32(val, ipa->reg_virt + offset);  in ipa_hardware_config_tx()
240 static void ipa_hardware_config_clkon(struct ipa *ipa)  in ipa_hardware_config_clkon()  argument
242 	enum ipa_version version = ipa->version;  in ipa_hardware_config_clkon()
253 	reg = ipa_reg(ipa, CLKON_CFG);  in ipa_hardware_config_clkon()
257 	} else {	/* IPA v4.0+ */  in ipa_hardware_config_clkon()
263 	iowrite32(val, ipa->reg_virt + reg_offset(reg));  in ipa_hardware_config_clkon()
266 /* Configure bus access behavior for IPA components */
267 static void ipa_hardware_config_comp(struct ipa *ipa)  in ipa_hardware_config_comp()  argument
273 	/* Nothing to configure prior to IPA v4.0 */  in ipa_hardware_config_comp()
274 	if (ipa->version < IPA_VERSION_4_0)  in ipa_hardware_config_comp()
277 	reg = ipa_reg(ipa, COMP_CFG);  in ipa_hardware_config_comp()
280 	val = ioread32(ipa->reg_virt + offset);  in ipa_hardware_config_comp()
282 	if (ipa->version == IPA_VERSION_4_0) {  in ipa_hardware_config_comp()
286 	} else if (ipa->version < IPA_VERSION_4_5) {  in ipa_hardware_config_comp()
289 		/* For IPA v4.5+ FULL_FLUSH_WAIT_RS_CLOSURE_EN is 0 */  in ipa_hardware_config_comp()
295 	iowrite32(val, ipa->reg_virt + offset);  in ipa_hardware_config_comp()
300 ipa_hardware_config_qsb(struct ipa *ipa, const struct ipa_data *data)  in ipa_hardware_config_qsb()  argument
313 	reg = ipa_reg(ipa, QSB_MAX_WRITES);  in ipa_hardware_config_qsb()
319 	iowrite32(val, ipa->reg_virt + reg_offset(reg));  in ipa_hardware_config_qsb()
322 	reg = ipa_reg(ipa, QSB_MAX_READS);  in ipa_hardware_config_qsb()
325 	if (ipa->version >= IPA_VERSION_4_0)  in ipa_hardware_config_qsb()
330 		if (ipa->version >= IPA_VERSION_4_0)  in ipa_hardware_config_qsb()
335 	iowrite32(val, ipa->reg_virt + reg_offset(reg));  in ipa_hardware_config_qsb()
352 /* IPA uses unified Qtime starting at IPA v4.5, implementing various
353  * timestamps and timers independent of the IPA core clock rate.  The
357  * For IPA timestamps (tag, NAT, data path logging) a lower resolution
365  * timer ticks at a configurable frequency.  IPA timers (such as
369 static void ipa_qtime_config(struct ipa *ipa)  in ipa_qtime_config()  argument
376 	reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG);  in ipa_qtime_config()
377 	iowrite32(0, ipa->reg_virt + reg_offset(reg));  in ipa_qtime_config()
379 	reg = ipa_reg(ipa, QTIME_TIMESTAMP_CFG);  in ipa_qtime_config()
380 	if (ipa->version < IPA_VERSION_5_5) {  in ipa_qtime_config()
389 	iowrite32(val, ipa->reg_virt + reg_offset(reg));  in ipa_qtime_config()
392 	reg = ipa_reg(ipa, TIMERS_PULSE_GRAN_CFG);  in ipa_qtime_config()
395 	if (ipa->version >= IPA_VERSION_5_0) {  in ipa_qtime_config()
402 	iowrite32(val, ipa->reg_virt + reg_offset(reg));  in ipa_qtime_config()
405 	reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG);  in ipa_qtime_config()
410 	iowrite32(val, ipa->reg_virt + offset);  in ipa_qtime_config()
415 	iowrite32(val, ipa->reg_virt + offset);  in ipa_qtime_config()
418 /* Before IPA v4.5 timing is controlled by a counter register */
419 static void ipa_hardware_config_counter(struct ipa *ipa)  in ipa_hardware_config_counter()  argument
425 	reg = ipa_reg(ipa, COUNTER_CFG);  in ipa_hardware_config_counter()
428 	iowrite32(val, ipa->reg_virt + reg_offset(reg));  in ipa_hardware_config_counter()
431 static void ipa_hardware_config_timing(struct ipa *ipa)  in ipa_hardware_config_timing()  argument
433 	if (ipa->version < IPA_VERSION_4_5)  in ipa_hardware_config_timing()
434 		ipa_hardware_config_counter(ipa);  in ipa_hardware_config_timing()
436 		ipa_qtime_config(ipa);  in ipa_hardware_config_timing()
439 static void ipa_hardware_config_hashing(struct ipa *ipa)  in ipa_hardware_config_hashing()  argument
443 	/* Other than IPA v4.2, all versions enable "hashing".  Starting  in ipa_hardware_config_hashing()
444 	 * with IPA v5.0, the filter and router tables are implemented  in ipa_hardware_config_hashing()
448 	if (ipa->version != IPA_VERSION_4_2)  in ipa_hardware_config_hashing()
451 	/* IPA v4.2 does not support hashed tables, so disable them */  in ipa_hardware_config_hashing()
452 	reg = ipa_reg(ipa, FILT_ROUT_HASH_EN);  in ipa_hardware_config_hashing()
457 	iowrite32(0, ipa->reg_virt + reg_offset(reg));  in ipa_hardware_config_hashing()
460 static void ipa_idle_indication_cfg(struct ipa *ipa,  in ipa_idle_indication_cfg()  argument
467 	if (ipa->version < IPA_VERSION_3_5_1)  in ipa_idle_indication_cfg()
470 	reg = ipa_reg(ipa, IDLE_INDICATION_CFG);  in ipa_idle_indication_cfg()
476 	iowrite32(val, ipa->reg_virt + reg_offset(reg));  in ipa_idle_indication_cfg()
480  * ipa_hardware_dcd_config() - Enable dynamic clock division on IPA
481  * @ipa:	IPA pointer
483  * Configures when the IPA signals it is idle to the global clock
487 static void ipa_hardware_dcd_config(struct ipa *ipa)  in ipa_hardware_dcd_config()  argument
489 	/* Recommended values for IPA 3.5 and later according to IPA HPG */  in ipa_hardware_dcd_config()
490 	ipa_idle_indication_cfg(ipa, 256, false);  in ipa_hardware_dcd_config()
493 static void ipa_hardware_dcd_deconfig(struct ipa *ipa)  in ipa_hardware_dcd_deconfig()  argument
496 	ipa_idle_indication_cfg(ipa, 0, true);  in ipa_hardware_dcd_deconfig()
501  * @ipa:	IPA pointer
502  * @data:	IPA configuration data
504 static void ipa_hardware_config(struct ipa *ipa, const struct ipa_data *data)  in ipa_hardware_config()  argument
506 	ipa_hardware_config_bcr(ipa, data);  in ipa_hardware_config()
507 	ipa_hardware_config_tx(ipa);  in ipa_hardware_config()
508 	ipa_hardware_config_clkon(ipa);  in ipa_hardware_config()
509 	ipa_hardware_config_comp(ipa);  in ipa_hardware_config()
510 	ipa_hardware_config_qsb(ipa, data);  in ipa_hardware_config()
511 	ipa_hardware_config_timing(ipa);  in ipa_hardware_config()
512 	ipa_hardware_config_hashing(ipa);  in ipa_hardware_config()
513 	ipa_hardware_dcd_config(ipa);  in ipa_hardware_config()
518  * @ipa:	IPA pointer
522 static void ipa_hardware_deconfig(struct ipa *ipa)  in ipa_hardware_deconfig()  argument
525 	ipa_hardware_dcd_deconfig(ipa);  in ipa_hardware_deconfig()
529  * ipa_config() - Configure IPA hardware
530  * @ipa:	IPA pointer
531  * @data:	IPA configuration data
533  * Perform initialization requiring IPA power to be enabled.
535 static int ipa_config(struct ipa *ipa, const struct ipa_data *data)  in ipa_config()  argument
539 	ipa_hardware_config(ipa, data);  in ipa_config()
541 	ret = ipa_mem_config(ipa);  in ipa_config()
545 	ipa->interrupt = ipa_interrupt_config(ipa);  in ipa_config()
546 	if (IS_ERR(ipa->interrupt)) {  in ipa_config()
547 		ret = PTR_ERR(ipa->interrupt);  in ipa_config()
548 		ipa->interrupt = NULL;  in ipa_config()
552 	ipa_uc_config(ipa);  in ipa_config()
554 	ret = ipa_endpoint_config(ipa);  in ipa_config()
558 	ipa_table_config(ipa);		/* No deconfig required */  in ipa_config()
561 	ret = ipa_resource_config(ipa, data->resource_data);  in ipa_config()
565 	ret = ipa_modem_config(ipa);  in ipa_config()
572 	ipa_endpoint_deconfig(ipa);  in ipa_config()
574 	ipa_uc_deconfig(ipa);  in ipa_config()
575 	ipa_interrupt_deconfig(ipa->interrupt);  in ipa_config()
576 	ipa->interrupt = NULL;  in ipa_config()
578 	ipa_mem_deconfig(ipa);  in ipa_config()
580 	ipa_hardware_deconfig(ipa);  in ipa_config()
587  * @ipa:	IPA pointer
589 static void ipa_deconfig(struct ipa *ipa)  in ipa_deconfig()  argument
591 	ipa_modem_deconfig(ipa);  in ipa_deconfig()
592 	ipa_endpoint_deconfig(ipa);  in ipa_deconfig()
593 	ipa_uc_deconfig(ipa);  in ipa_deconfig()
594 	ipa_interrupt_deconfig(ipa->interrupt);  in ipa_deconfig()
595 	ipa->interrupt = NULL;  in ipa_deconfig()
596 	ipa_mem_deconfig(ipa);  in ipa_deconfig()
597 	ipa_hardware_deconfig(ipa);  in ipa_deconfig()
663 		.compatible	= "qcom,msm8998-ipa",
667 		.compatible	= "qcom,sdm845-ipa",
671 		.compatible	= "qcom,sc7180-ipa",
675 		.compatible	= "qcom,sdx55-ipa",
679 		.compatible	= "qcom,sm6350-ipa",
683 		.compatible	= "qcom,sm8350-ipa",
687 		.compatible	= "qcom,sc7280-ipa",
691 		.compatible	= "qcom,sdx65-ipa",
695 		.compatible	= "qcom,sm8550-ipa",
786  * ipa_probe() - IPA platform driver probe function
792  * This is the main entry point for the IPA driver.  Initialization proceeds
795  *     access to the IPA hardware.
796  *   - The "config" stage requires IPA power to be active so IPA registers
797  *     can be accessed, but does not require the use of IPA immediate commands.
798  *   - The "setup" stage uses IPA immediate commands, and so requires the GSI
806  * to the AP; this triggers the start if IPA setup.
814 	struct ipa *ipa;  in ipa_probe()  local
827 		dev_err(dev, "unsupported IPA version %u\n", data->version);  in ipa_probe()
849 	/* No more EPROBE_DEFER.  Allocate and initialize the IPA structure */  in ipa_probe()
850 	ipa = kzalloc(sizeof(*ipa), GFP_KERNEL);  in ipa_probe()
851 	if (!ipa) {  in ipa_probe()
856 	ipa->pdev = pdev;  in ipa_probe()
857 	dev_set_drvdata(dev, ipa);  in ipa_probe()
858 	ipa->power = power;  in ipa_probe()
859 	ipa->version = data->version;  in ipa_probe()
860 	ipa->modem_route_count = data->modem_route_count;  in ipa_probe()
861 	init_completion(&ipa->completion);  in ipa_probe()
863 	ret = ipa_reg_init(ipa);  in ipa_probe()
867 	ret = ipa_mem_init(ipa, data->mem_data);  in ipa_probe()
871 	ret = gsi_init(&ipa->gsi, pdev, ipa->version, data->endpoint_count,  in ipa_probe()
877 	ret = ipa_endpoint_init(ipa, data->endpoint_count, data->endpoint_data);  in ipa_probe()
881 	ret = ipa_table_init(ipa);  in ipa_probe()
885 	ret = ipa_smp2p_init(ipa, loader == IPA_LOADER_MODEM);  in ipa_probe()
894 	ret = ipa_config(ipa, data);  in ipa_probe()
898 	dev_info(dev, "IPA driver initialized");  in ipa_probe()
914 	ret = ipa_setup(ipa);  in ipa_probe()
924 	ipa_deconfig(ipa);  in ipa_probe()
927 	ipa_smp2p_exit(ipa);  in ipa_probe()
929 	ipa_table_exit(ipa);  in ipa_probe()
931 	ipa_endpoint_exit(ipa);  in ipa_probe()
933 	gsi_exit(&ipa->gsi);  in ipa_probe()
935 	ipa_mem_exit(ipa);  in ipa_probe()
937 	ipa_reg_exit(ipa);  in ipa_probe()
939 	kfree(ipa);  in ipa_probe()
948 	struct ipa *ipa = dev_get_drvdata(&pdev->dev);  in ipa_remove()  local
949 	struct ipa_power *power = ipa->power;  in ipa_remove()
956 	ipa_smp2p_irq_disable_setup(ipa);  in ipa_remove()
962 	if (ipa->setup_complete) {  in ipa_remove()
963 		ret = ipa_modem_stop(ipa);  in ipa_remove()
967 			ret = ipa_modem_stop(ipa);  in ipa_remove()
980 		ipa_teardown(ipa);  in ipa_remove()
983 	ipa_deconfig(ipa);  in ipa_remove()
986 	ipa_smp2p_exit(ipa);  in ipa_remove()
987 	ipa_table_exit(ipa);  in ipa_remove()
988 	ipa_endpoint_exit(ipa);  in ipa_remove()
989 	gsi_exit(&ipa->gsi);  in ipa_remove()
990 	ipa_mem_exit(ipa);  in ipa_remove()
991 	ipa_reg_exit(ipa);  in ipa_remove()
992 	kfree(ipa);  in ipa_remove()
995 	dev_info(dev, "IPA driver removed");  in ipa_remove()
1011 		.name		= "ipa",