1 /*
2  * drivers/base/power/domain.c - Common code related to device power domains.
3  *
4  * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
5  *
6  * This file is released under the GPLv2.
7  */
8 
9 #include <linux/init.h>
10 #include <linux/kernel.h>
11 #include <linux/io.h>
12 #include <linux/pm_runtime.h>
13 #include <linux/pm_domain.h>
14 #include <linux/slab.h>
15 #include <linux/err.h>
16 #include <linux/sched.h>
17 #include <linux/suspend.h>
18 #include <linux/export.h>
19 
20 #define GENPD_DEV_CALLBACK(genpd, type, callback, dev)		\
21 ({								\
22 	type (*__routine)(struct device *__d); 			\
23 	type __ret = (type)0;					\
24 								\
25 	__routine = genpd->dev_ops.callback; 			\
26 	if (__routine) {					\
27 		__ret = __routine(dev); 			\
28 	} else {						\
29 		__routine = dev_gpd_data(dev)->ops.callback;	\
30 		if (__routine) 					\
31 			__ret = __routine(dev);			\
32 	}							\
33 	__ret;							\
34 })
35 
36 #define GENPD_DEV_TIMED_CALLBACK(genpd, type, callback, dev, field, name)	\
37 ({										\
38 	ktime_t __start = ktime_get();						\
39 	type __retval = GENPD_DEV_CALLBACK(genpd, type, callback, dev);		\
40 	s64 __elapsed = ktime_to_ns(ktime_sub(ktime_get(), __start));		\
41 	struct generic_pm_domain_data *__gpd_data = dev_gpd_data(dev);		\
42 	if (__elapsed > __gpd_data->td.field) {					\
43 		__gpd_data->td.field = __elapsed;				\
44 		dev_warn(dev, name " latency exceeded, new value %lld ns\n",	\
45 			__elapsed);						\
46 	}									\
47 	__retval;								\
48 })
49 
50 static LIST_HEAD(gpd_list);
51 static DEFINE_MUTEX(gpd_list_lock);
52 
53 #ifdef CONFIG_PM
54 
dev_to_genpd(struct device * dev)55 struct generic_pm_domain *dev_to_genpd(struct device *dev)
56 {
57 	if (IS_ERR_OR_NULL(dev->pm_domain))
58 		return ERR_PTR(-EINVAL);
59 
60 	return pd_to_genpd(dev->pm_domain);
61 }
62 
genpd_stop_dev(struct generic_pm_domain * genpd,struct device * dev)63 static int genpd_stop_dev(struct generic_pm_domain *genpd, struct device *dev)
64 {
65 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, stop, dev,
66 					stop_latency_ns, "stop");
67 }
68 
genpd_start_dev(struct generic_pm_domain * genpd,struct device * dev)69 static int genpd_start_dev(struct generic_pm_domain *genpd, struct device *dev)
70 {
71 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, start, dev,
72 					start_latency_ns, "start");
73 }
74 
genpd_save_dev(struct generic_pm_domain * genpd,struct device * dev)75 static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev)
76 {
77 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, save_state, dev,
78 					save_state_latency_ns, "state save");
79 }
80 
genpd_restore_dev(struct generic_pm_domain * genpd,struct device * dev)81 static int genpd_restore_dev(struct generic_pm_domain *genpd, struct device *dev)
82 {
83 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, restore_state, dev,
84 					restore_state_latency_ns,
85 					"state restore");
86 }
87 
genpd_sd_counter_dec(struct generic_pm_domain * genpd)88 static bool genpd_sd_counter_dec(struct generic_pm_domain *genpd)
89 {
90 	bool ret = false;
91 
92 	if (!WARN_ON(atomic_read(&genpd->sd_count) == 0))
93 		ret = !!atomic_dec_and_test(&genpd->sd_count);
94 
95 	return ret;
96 }
97 
genpd_sd_counter_inc(struct generic_pm_domain * genpd)98 static void genpd_sd_counter_inc(struct generic_pm_domain *genpd)
99 {
100 	atomic_inc(&genpd->sd_count);
101 	smp_mb__after_atomic_inc();
102 }
103 
genpd_acquire_lock(struct generic_pm_domain * genpd)104 static void genpd_acquire_lock(struct generic_pm_domain *genpd)
105 {
106 	DEFINE_WAIT(wait);
107 
108 	mutex_lock(&genpd->lock);
109 	/*
110 	 * Wait for the domain to transition into either the active,
111 	 * or the power off state.
112 	 */
113 	for (;;) {
114 		prepare_to_wait(&genpd->status_wait_queue, &wait,
115 				TASK_UNINTERRUPTIBLE);
116 		if (genpd->status == GPD_STATE_ACTIVE
117 		    || genpd->status == GPD_STATE_POWER_OFF)
118 			break;
119 		mutex_unlock(&genpd->lock);
120 
121 		schedule();
122 
123 		mutex_lock(&genpd->lock);
124 	}
125 	finish_wait(&genpd->status_wait_queue, &wait);
126 }
127 
genpd_release_lock(struct generic_pm_domain * genpd)128 static void genpd_release_lock(struct generic_pm_domain *genpd)
129 {
130 	mutex_unlock(&genpd->lock);
131 }
132 
genpd_set_active(struct generic_pm_domain * genpd)133 static void genpd_set_active(struct generic_pm_domain *genpd)
134 {
135 	if (genpd->resume_count == 0)
136 		genpd->status = GPD_STATE_ACTIVE;
137 }
138 
139 /**
140  * __pm_genpd_poweron - Restore power to a given PM domain and its masters.
141  * @genpd: PM domain to power up.
142  *
143  * Restore power to @genpd and all of its masters so that it is possible to
144  * resume a device belonging to it.
145  */
__pm_genpd_poweron(struct generic_pm_domain * genpd)146 int __pm_genpd_poweron(struct generic_pm_domain *genpd)
147 	__releases(&genpd->lock) __acquires(&genpd->lock)
148 {
149 	struct gpd_link *link;
150 	DEFINE_WAIT(wait);
151 	int ret = 0;
152 
153 	/* If the domain's master is being waited for, we have to wait too. */
154 	for (;;) {
155 		prepare_to_wait(&genpd->status_wait_queue, &wait,
156 				TASK_UNINTERRUPTIBLE);
157 		if (genpd->status != GPD_STATE_WAIT_MASTER)
158 			break;
159 		mutex_unlock(&genpd->lock);
160 
161 		schedule();
162 
163 		mutex_lock(&genpd->lock);
164 	}
165 	finish_wait(&genpd->status_wait_queue, &wait);
166 
167 	if (genpd->status == GPD_STATE_ACTIVE
168 	    || (genpd->prepared_count > 0 && genpd->suspend_power_off))
169 		return 0;
170 
171 	if (genpd->status != GPD_STATE_POWER_OFF) {
172 		genpd_set_active(genpd);
173 		return 0;
174 	}
175 
176 	/*
177 	 * The list is guaranteed not to change while the loop below is being
178 	 * executed, unless one of the masters' .power_on() callbacks fiddles
179 	 * with it.
180 	 */
181 	list_for_each_entry(link, &genpd->slave_links, slave_node) {
182 		genpd_sd_counter_inc(link->master);
183 		genpd->status = GPD_STATE_WAIT_MASTER;
184 
185 		mutex_unlock(&genpd->lock);
186 
187 		ret = pm_genpd_poweron(link->master);
188 
189 		mutex_lock(&genpd->lock);
190 
191 		/*
192 		 * The "wait for parent" status is guaranteed not to change
193 		 * while the master is powering on.
194 		 */
195 		genpd->status = GPD_STATE_POWER_OFF;
196 		wake_up_all(&genpd->status_wait_queue);
197 		if (ret) {
198 			genpd_sd_counter_dec(link->master);
199 			goto err;
200 		}
201 	}
202 
203 	if (genpd->power_on) {
204 		ktime_t time_start = ktime_get();
205 		s64 elapsed_ns;
206 
207 		ret = genpd->power_on(genpd);
208 		if (ret)
209 			goto err;
210 
211 		elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
212 		if (elapsed_ns > genpd->power_on_latency_ns) {
213 			genpd->power_on_latency_ns = elapsed_ns;
214 			if (genpd->name)
215 				pr_warning("%s: Power-on latency exceeded, "
216 					"new value %lld ns\n", genpd->name,
217 					elapsed_ns);
218 		}
219 	}
220 
221 	genpd_set_active(genpd);
222 
223 	return 0;
224 
225  err:
226 	list_for_each_entry_continue_reverse(link, &genpd->slave_links, slave_node)
227 		genpd_sd_counter_dec(link->master);
228 
229 	return ret;
230 }
231 
232 /**
233  * pm_genpd_poweron - Restore power to a given PM domain and its masters.
234  * @genpd: PM domain to power up.
235  */
pm_genpd_poweron(struct generic_pm_domain * genpd)236 int pm_genpd_poweron(struct generic_pm_domain *genpd)
237 {
238 	int ret;
239 
240 	mutex_lock(&genpd->lock);
241 	ret = __pm_genpd_poweron(genpd);
242 	mutex_unlock(&genpd->lock);
243 	return ret;
244 }
245 
246 #endif /* CONFIG_PM */
247 
248 #ifdef CONFIG_PM_RUNTIME
249 
250 /**
251  * __pm_genpd_save_device - Save the pre-suspend state of a device.
252  * @pdd: Domain data of the device to save the state of.
253  * @genpd: PM domain the device belongs to.
254  */
__pm_genpd_save_device(struct pm_domain_data * pdd,struct generic_pm_domain * genpd)255 static int __pm_genpd_save_device(struct pm_domain_data *pdd,
256 				  struct generic_pm_domain *genpd)
257 	__releases(&genpd->lock) __acquires(&genpd->lock)
258 {
259 	struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd);
260 	struct device *dev = pdd->dev;
261 	int ret = 0;
262 
263 	if (gpd_data->need_restore)
264 		return 0;
265 
266 	mutex_unlock(&genpd->lock);
267 
268 	genpd_start_dev(genpd, dev);
269 	ret = genpd_save_dev(genpd, dev);
270 	genpd_stop_dev(genpd, dev);
271 
272 	mutex_lock(&genpd->lock);
273 
274 	if (!ret)
275 		gpd_data->need_restore = true;
276 
277 	return ret;
278 }
279 
280 /**
281  * __pm_genpd_restore_device - Restore the pre-suspend state of a device.
282  * @pdd: Domain data of the device to restore the state of.
283  * @genpd: PM domain the device belongs to.
284  */
__pm_genpd_restore_device(struct pm_domain_data * pdd,struct generic_pm_domain * genpd)285 static void __pm_genpd_restore_device(struct pm_domain_data *pdd,
286 				      struct generic_pm_domain *genpd)
287 	__releases(&genpd->lock) __acquires(&genpd->lock)
288 {
289 	struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd);
290 	struct device *dev = pdd->dev;
291 
292 	if (!gpd_data->need_restore)
293 		return;
294 
295 	mutex_unlock(&genpd->lock);
296 
297 	genpd_start_dev(genpd, dev);
298 	genpd_restore_dev(genpd, dev);
299 	genpd_stop_dev(genpd, dev);
300 
301 	mutex_lock(&genpd->lock);
302 
303 	gpd_data->need_restore = false;
304 }
305 
306 /**
307  * genpd_abort_poweroff - Check if a PM domain power off should be aborted.
308  * @genpd: PM domain to check.
309  *
310  * Return true if a PM domain's status changed to GPD_STATE_ACTIVE during
311  * a "power off" operation, which means that a "power on" has occured in the
312  * meantime, or if its resume_count field is different from zero, which means
313  * that one of its devices has been resumed in the meantime.
314  */
genpd_abort_poweroff(struct generic_pm_domain * genpd)315 static bool genpd_abort_poweroff(struct generic_pm_domain *genpd)
316 {
317 	return genpd->status == GPD_STATE_WAIT_MASTER
318 		|| genpd->status == GPD_STATE_ACTIVE || genpd->resume_count > 0;
319 }
320 
321 /**
322  * genpd_queue_power_off_work - Queue up the execution of pm_genpd_poweroff().
323  * @genpd: PM domait to power off.
324  *
325  * Queue up the execution of pm_genpd_poweroff() unless it's already been done
326  * before.
327  */
genpd_queue_power_off_work(struct generic_pm_domain * genpd)328 void genpd_queue_power_off_work(struct generic_pm_domain *genpd)
329 {
330 	if (!work_pending(&genpd->power_off_work))
331 		queue_work(pm_wq, &genpd->power_off_work);
332 }
333 
334 /**
335  * pm_genpd_poweroff - Remove power from a given PM domain.
336  * @genpd: PM domain to power down.
337  *
338  * If all of the @genpd's devices have been suspended and all of its subdomains
339  * have been powered down, run the runtime suspend callbacks provided by all of
340  * the @genpd's devices' drivers and remove power from @genpd.
341  */
pm_genpd_poweroff(struct generic_pm_domain * genpd)342 static int pm_genpd_poweroff(struct generic_pm_domain *genpd)
343 	__releases(&genpd->lock) __acquires(&genpd->lock)
344 {
345 	struct pm_domain_data *pdd;
346 	struct gpd_link *link;
347 	unsigned int not_suspended;
348 	int ret = 0;
349 
350  start:
351 	/*
352 	 * Do not try to power off the domain in the following situations:
353 	 * (1) The domain is already in the "power off" state.
354 	 * (2) The domain is waiting for its master to power up.
355 	 * (3) One of the domain's devices is being resumed right now.
356 	 * (4) System suspend is in progress.
357 	 */
358 	if (genpd->status == GPD_STATE_POWER_OFF
359 	    || genpd->status == GPD_STATE_WAIT_MASTER
360 	    || genpd->resume_count > 0 || genpd->prepared_count > 0)
361 		return 0;
362 
363 	if (atomic_read(&genpd->sd_count) > 0)
364 		return -EBUSY;
365 
366 	not_suspended = 0;
367 	list_for_each_entry(pdd, &genpd->dev_list, list_node)
368 		if (pdd->dev->driver && (!pm_runtime_suspended(pdd->dev)
369 		    || pdd->dev->power.irq_safe))
370 			not_suspended++;
371 
372 	if (not_suspended > genpd->in_progress)
373 		return -EBUSY;
374 
375 	if (genpd->poweroff_task) {
376 		/*
377 		 * Another instance of pm_genpd_poweroff() is executing
378 		 * callbacks, so tell it to start over and return.
379 		 */
380 		genpd->status = GPD_STATE_REPEAT;
381 		return 0;
382 	}
383 
384 	if (genpd->gov && genpd->gov->power_down_ok) {
385 		if (!genpd->gov->power_down_ok(&genpd->domain))
386 			return -EAGAIN;
387 	}
388 
389 	genpd->status = GPD_STATE_BUSY;
390 	genpd->poweroff_task = current;
391 
392 	list_for_each_entry_reverse(pdd, &genpd->dev_list, list_node) {
393 		ret = atomic_read(&genpd->sd_count) == 0 ?
394 			__pm_genpd_save_device(pdd, genpd) : -EBUSY;
395 
396 		if (genpd_abort_poweroff(genpd))
397 			goto out;
398 
399 		if (ret) {
400 			genpd_set_active(genpd);
401 			goto out;
402 		}
403 
404 		if (genpd->status == GPD_STATE_REPEAT) {
405 			genpd->poweroff_task = NULL;
406 			goto start;
407 		}
408 	}
409 
410 	if (genpd->power_off) {
411 		ktime_t time_start;
412 		s64 elapsed_ns;
413 
414 		if (atomic_read(&genpd->sd_count) > 0) {
415 			ret = -EBUSY;
416 			goto out;
417 		}
418 
419 		time_start = ktime_get();
420 
421 		/*
422 		 * If sd_count > 0 at this point, one of the subdomains hasn't
423 		 * managed to call pm_genpd_poweron() for the master yet after
424 		 * incrementing it.  In that case pm_genpd_poweron() will wait
425 		 * for us to drop the lock, so we can call .power_off() and let
426 		 * the pm_genpd_poweron() restore power for us (this shouldn't
427 		 * happen very often).
428 		 */
429 		ret = genpd->power_off(genpd);
430 		if (ret == -EBUSY) {
431 			genpd_set_active(genpd);
432 			goto out;
433 		}
434 
435 		elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
436 		if (elapsed_ns > genpd->power_off_latency_ns) {
437 			genpd->power_off_latency_ns = elapsed_ns;
438 			if (genpd->name)
439 				pr_warning("%s: Power-off latency exceeded, "
440 					"new value %lld ns\n", genpd->name,
441 					elapsed_ns);
442 		}
443 	}
444 
445 	genpd->status = GPD_STATE_POWER_OFF;
446 	genpd->power_off_time = ktime_get();
447 
448 	/* Update PM QoS information for devices in the domain. */
449 	list_for_each_entry_reverse(pdd, &genpd->dev_list, list_node) {
450 		struct gpd_timing_data *td = &to_gpd_data(pdd)->td;
451 
452 		pm_runtime_update_max_time_suspended(pdd->dev,
453 					td->start_latency_ns +
454 					td->restore_state_latency_ns +
455 					genpd->power_on_latency_ns);
456 	}
457 
458 	list_for_each_entry(link, &genpd->slave_links, slave_node) {
459 		genpd_sd_counter_dec(link->master);
460 		genpd_queue_power_off_work(link->master);
461 	}
462 
463  out:
464 	genpd->poweroff_task = NULL;
465 	wake_up_all(&genpd->status_wait_queue);
466 	return ret;
467 }
468 
469 /**
470  * genpd_power_off_work_fn - Power off PM domain whose subdomain count is 0.
471  * @work: Work structure used for scheduling the execution of this function.
472  */
genpd_power_off_work_fn(struct work_struct * work)473 static void genpd_power_off_work_fn(struct work_struct *work)
474 {
475 	struct generic_pm_domain *genpd;
476 
477 	genpd = container_of(work, struct generic_pm_domain, power_off_work);
478 
479 	genpd_acquire_lock(genpd);
480 	pm_genpd_poweroff(genpd);
481 	genpd_release_lock(genpd);
482 }
483 
484 /**
485  * pm_genpd_runtime_suspend - Suspend a device belonging to I/O PM domain.
486  * @dev: Device to suspend.
487  *
488  * Carry out a runtime suspend of a device under the assumption that its
489  * pm_domain field points to the domain member of an object of type
490  * struct generic_pm_domain representing a PM domain consisting of I/O devices.
491  */
pm_genpd_runtime_suspend(struct device * dev)492 static int pm_genpd_runtime_suspend(struct device *dev)
493 {
494 	struct generic_pm_domain *genpd;
495 	bool (*stop_ok)(struct device *__dev);
496 	int ret;
497 
498 	dev_dbg(dev, "%s()\n", __func__);
499 
500 	genpd = dev_to_genpd(dev);
501 	if (IS_ERR(genpd))
502 		return -EINVAL;
503 
504 	might_sleep_if(!genpd->dev_irq_safe);
505 
506 	stop_ok = genpd->gov ? genpd->gov->stop_ok : NULL;
507 	if (stop_ok && !stop_ok(dev))
508 		return -EBUSY;
509 
510 	ret = genpd_stop_dev(genpd, dev);
511 	if (ret)
512 		return ret;
513 
514 	pm_runtime_update_max_time_suspended(dev,
515 				dev_gpd_data(dev)->td.start_latency_ns);
516 
517 	/*
518 	 * If power.irq_safe is set, this routine will be run with interrupts
519 	 * off, so it can't use mutexes.
520 	 */
521 	if (dev->power.irq_safe)
522 		return 0;
523 
524 	mutex_lock(&genpd->lock);
525 	genpd->in_progress++;
526 	pm_genpd_poweroff(genpd);
527 	genpd->in_progress--;
528 	mutex_unlock(&genpd->lock);
529 
530 	return 0;
531 }
532 
533 /**
534  * pm_genpd_runtime_resume - Resume a device belonging to I/O PM domain.
535  * @dev: Device to resume.
536  *
537  * Carry out a runtime resume of a device under the assumption that its
538  * pm_domain field points to the domain member of an object of type
539  * struct generic_pm_domain representing a PM domain consisting of I/O devices.
540  */
pm_genpd_runtime_resume(struct device * dev)541 static int pm_genpd_runtime_resume(struct device *dev)
542 {
543 	struct generic_pm_domain *genpd;
544 	DEFINE_WAIT(wait);
545 	int ret;
546 
547 	dev_dbg(dev, "%s()\n", __func__);
548 
549 	genpd = dev_to_genpd(dev);
550 	if (IS_ERR(genpd))
551 		return -EINVAL;
552 
553 	might_sleep_if(!genpd->dev_irq_safe);
554 
555 	/* If power.irq_safe, the PM domain is never powered off. */
556 	if (dev->power.irq_safe)
557 		goto out;
558 
559 	mutex_lock(&genpd->lock);
560 	ret = __pm_genpd_poweron(genpd);
561 	if (ret) {
562 		mutex_unlock(&genpd->lock);
563 		return ret;
564 	}
565 	genpd->status = GPD_STATE_BUSY;
566 	genpd->resume_count++;
567 	for (;;) {
568 		prepare_to_wait(&genpd->status_wait_queue, &wait,
569 				TASK_UNINTERRUPTIBLE);
570 		/*
571 		 * If current is the powering off task, we have been called
572 		 * reentrantly from one of the device callbacks, so we should
573 		 * not wait.
574 		 */
575 		if (!genpd->poweroff_task || genpd->poweroff_task == current)
576 			break;
577 		mutex_unlock(&genpd->lock);
578 
579 		schedule();
580 
581 		mutex_lock(&genpd->lock);
582 	}
583 	finish_wait(&genpd->status_wait_queue, &wait);
584 	__pm_genpd_restore_device(dev->power.subsys_data->domain_data, genpd);
585 	genpd->resume_count--;
586 	genpd_set_active(genpd);
587 	wake_up_all(&genpd->status_wait_queue);
588 	mutex_unlock(&genpd->lock);
589 
590  out:
591 	genpd_start_dev(genpd, dev);
592 
593 	return 0;
594 }
595 
596 /**
597  * pm_genpd_poweroff_unused - Power off all PM domains with no devices in use.
598  */
pm_genpd_poweroff_unused(void)599 void pm_genpd_poweroff_unused(void)
600 {
601 	struct generic_pm_domain *genpd;
602 
603 	mutex_lock(&gpd_list_lock);
604 
605 	list_for_each_entry(genpd, &gpd_list, gpd_list_node)
606 		genpd_queue_power_off_work(genpd);
607 
608 	mutex_unlock(&gpd_list_lock);
609 }
610 
611 #else
612 
genpd_power_off_work_fn(struct work_struct * work)613 static inline void genpd_power_off_work_fn(struct work_struct *work) {}
614 
615 #define pm_genpd_runtime_suspend	NULL
616 #define pm_genpd_runtime_resume		NULL
617 
618 #endif /* CONFIG_PM_RUNTIME */
619 
620 #ifdef CONFIG_PM_SLEEP
621 
genpd_dev_active_wakeup(struct generic_pm_domain * genpd,struct device * dev)622 static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd,
623 				    struct device *dev)
624 {
625 	return GENPD_DEV_CALLBACK(genpd, bool, active_wakeup, dev);
626 }
627 
genpd_suspend_dev(struct generic_pm_domain * genpd,struct device * dev)628 static int genpd_suspend_dev(struct generic_pm_domain *genpd, struct device *dev)
629 {
630 	return GENPD_DEV_CALLBACK(genpd, int, suspend, dev);
631 }
632 
genpd_suspend_late(struct generic_pm_domain * genpd,struct device * dev)633 static int genpd_suspend_late(struct generic_pm_domain *genpd, struct device *dev)
634 {
635 	return GENPD_DEV_CALLBACK(genpd, int, suspend_late, dev);
636 }
637 
genpd_resume_early(struct generic_pm_domain * genpd,struct device * dev)638 static int genpd_resume_early(struct generic_pm_domain *genpd, struct device *dev)
639 {
640 	return GENPD_DEV_CALLBACK(genpd, int, resume_early, dev);
641 }
642 
genpd_resume_dev(struct generic_pm_domain * genpd,struct device * dev)643 static int genpd_resume_dev(struct generic_pm_domain *genpd, struct device *dev)
644 {
645 	return GENPD_DEV_CALLBACK(genpd, int, resume, dev);
646 }
647 
genpd_freeze_dev(struct generic_pm_domain * genpd,struct device * dev)648 static int genpd_freeze_dev(struct generic_pm_domain *genpd, struct device *dev)
649 {
650 	return GENPD_DEV_CALLBACK(genpd, int, freeze, dev);
651 }
652 
genpd_freeze_late(struct generic_pm_domain * genpd,struct device * dev)653 static int genpd_freeze_late(struct generic_pm_domain *genpd, struct device *dev)
654 {
655 	return GENPD_DEV_CALLBACK(genpd, int, freeze_late, dev);
656 }
657 
genpd_thaw_early(struct generic_pm_domain * genpd,struct device * dev)658 static int genpd_thaw_early(struct generic_pm_domain *genpd, struct device *dev)
659 {
660 	return GENPD_DEV_CALLBACK(genpd, int, thaw_early, dev);
661 }
662 
genpd_thaw_dev(struct generic_pm_domain * genpd,struct device * dev)663 static int genpd_thaw_dev(struct generic_pm_domain *genpd, struct device *dev)
664 {
665 	return GENPD_DEV_CALLBACK(genpd, int, thaw, dev);
666 }
667 
668 /**
669  * pm_genpd_sync_poweroff - Synchronously power off a PM domain and its masters.
670  * @genpd: PM domain to power off, if possible.
671  *
672  * Check if the given PM domain can be powered off (during system suspend or
673  * hibernation) and do that if so.  Also, in that case propagate to its masters.
674  *
675  * This function is only called in "noirq" stages of system power transitions,
676  * so it need not acquire locks (all of the "noirq" callbacks are executed
677  * sequentially, so it is guaranteed that it will never run twice in parallel).
678  */
pm_genpd_sync_poweroff(struct generic_pm_domain * genpd)679 static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd)
680 {
681 	struct gpd_link *link;
682 
683 	if (genpd->status == GPD_STATE_POWER_OFF)
684 		return;
685 
686 	if (genpd->suspended_count != genpd->device_count
687 	    || atomic_read(&genpd->sd_count) > 0)
688 		return;
689 
690 	if (genpd->power_off)
691 		genpd->power_off(genpd);
692 
693 	genpd->status = GPD_STATE_POWER_OFF;
694 
695 	list_for_each_entry(link, &genpd->slave_links, slave_node) {
696 		genpd_sd_counter_dec(link->master);
697 		pm_genpd_sync_poweroff(link->master);
698 	}
699 }
700 
701 /**
702  * resume_needed - Check whether to resume a device before system suspend.
703  * @dev: Device to check.
704  * @genpd: PM domain the device belongs to.
705  *
706  * There are two cases in which a device that can wake up the system from sleep
707  * states should be resumed by pm_genpd_prepare(): (1) if the device is enabled
708  * to wake up the system and it has to remain active for this purpose while the
709  * system is in the sleep state and (2) if the device is not enabled to wake up
710  * the system from sleep states and it generally doesn't generate wakeup signals
711  * by itself (those signals are generated on its behalf by other parts of the
712  * system).  In the latter case it may be necessary to reconfigure the device's
713  * wakeup settings during system suspend, because it may have been set up to
714  * signal remote wakeup from the system's working state as needed by runtime PM.
715  * Return 'true' in either of the above cases.
716  */
resume_needed(struct device * dev,struct generic_pm_domain * genpd)717 static bool resume_needed(struct device *dev, struct generic_pm_domain *genpd)
718 {
719 	bool active_wakeup;
720 
721 	if (!device_can_wakeup(dev))
722 		return false;
723 
724 	active_wakeup = genpd_dev_active_wakeup(genpd, dev);
725 	return device_may_wakeup(dev) ? active_wakeup : !active_wakeup;
726 }
727 
728 /**
729  * pm_genpd_prepare - Start power transition of a device in a PM domain.
730  * @dev: Device to start the transition of.
731  *
732  * Start a power transition of a device (during a system-wide power transition)
733  * under the assumption that its pm_domain field points to the domain member of
734  * an object of type struct generic_pm_domain representing a PM domain
735  * consisting of I/O devices.
736  */
pm_genpd_prepare(struct device * dev)737 static int pm_genpd_prepare(struct device *dev)
738 {
739 	struct generic_pm_domain *genpd;
740 	int ret;
741 
742 	dev_dbg(dev, "%s()\n", __func__);
743 
744 	genpd = dev_to_genpd(dev);
745 	if (IS_ERR(genpd))
746 		return -EINVAL;
747 
748 	/*
749 	 * If a wakeup request is pending for the device, it should be woken up
750 	 * at this point and a system wakeup event should be reported if it's
751 	 * set up to wake up the system from sleep states.
752 	 */
753 	pm_runtime_get_noresume(dev);
754 	if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
755 		pm_wakeup_event(dev, 0);
756 
757 	if (pm_wakeup_pending()) {
758 		pm_runtime_put_sync(dev);
759 		return -EBUSY;
760 	}
761 
762 	if (resume_needed(dev, genpd))
763 		pm_runtime_resume(dev);
764 
765 	genpd_acquire_lock(genpd);
766 
767 	if (genpd->prepared_count++ == 0)
768 		genpd->suspend_power_off = genpd->status == GPD_STATE_POWER_OFF;
769 
770 	genpd_release_lock(genpd);
771 
772 	if (genpd->suspend_power_off) {
773 		pm_runtime_put_noidle(dev);
774 		return 0;
775 	}
776 
777 	/*
778 	 * The PM domain must be in the GPD_STATE_ACTIVE state at this point,
779 	 * so pm_genpd_poweron() will return immediately, but if the device
780 	 * is suspended (e.g. it's been stopped by genpd_stop_dev()), we need
781 	 * to make it operational.
782 	 */
783 	pm_runtime_resume(dev);
784 	__pm_runtime_disable(dev, false);
785 
786 	ret = pm_generic_prepare(dev);
787 	if (ret) {
788 		mutex_lock(&genpd->lock);
789 
790 		if (--genpd->prepared_count == 0)
791 			genpd->suspend_power_off = false;
792 
793 		mutex_unlock(&genpd->lock);
794 		pm_runtime_enable(dev);
795 	}
796 
797 	pm_runtime_put_sync(dev);
798 	return ret;
799 }
800 
801 /**
802  * pm_genpd_suspend - Suspend a device belonging to an I/O PM domain.
803  * @dev: Device to suspend.
804  *
805  * Suspend a device under the assumption that its pm_domain field points to the
806  * domain member of an object of type struct generic_pm_domain representing
807  * a PM domain consisting of I/O devices.
808  */
pm_genpd_suspend(struct device * dev)809 static int pm_genpd_suspend(struct device *dev)
810 {
811 	struct generic_pm_domain *genpd;
812 
813 	dev_dbg(dev, "%s()\n", __func__);
814 
815 	genpd = dev_to_genpd(dev);
816 	if (IS_ERR(genpd))
817 		return -EINVAL;
818 
819 	return genpd->suspend_power_off ? 0 : genpd_suspend_dev(genpd, dev);
820 }
821 
822 /**
823  * pm_genpd_suspend_noirq - Late suspend of a device from an I/O PM domain.
824  * @dev: Device to suspend.
825  *
826  * Carry out a late suspend of a device under the assumption that its
827  * pm_domain field points to the domain member of an object of type
828  * struct generic_pm_domain representing a PM domain consisting of I/O devices.
829  */
pm_genpd_suspend_noirq(struct device * dev)830 static int pm_genpd_suspend_noirq(struct device *dev)
831 {
832 	struct generic_pm_domain *genpd;
833 	int ret;
834 
835 	dev_dbg(dev, "%s()\n", __func__);
836 
837 	genpd = dev_to_genpd(dev);
838 	if (IS_ERR(genpd))
839 		return -EINVAL;
840 
841 	if (genpd->suspend_power_off)
842 		return 0;
843 
844 	ret = genpd_suspend_late(genpd, dev);
845 	if (ret)
846 		return ret;
847 
848 	if (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev))
849 		return 0;
850 
851 	genpd_stop_dev(genpd, dev);
852 
853 	/*
854 	 * Since all of the "noirq" callbacks are executed sequentially, it is
855 	 * guaranteed that this function will never run twice in parallel for
856 	 * the same PM domain, so it is not necessary to use locking here.
857 	 */
858 	genpd->suspended_count++;
859 	pm_genpd_sync_poweroff(genpd);
860 
861 	return 0;
862 }
863 
864 /**
865  * pm_genpd_resume_noirq - Early resume of a device from an I/O power domain.
866  * @dev: Device to resume.
867  *
868  * Carry out an early resume of a device under the assumption that its
869  * pm_domain field points to the domain member of an object of type
870  * struct generic_pm_domain representing a power domain consisting of I/O
871  * devices.
872  */
pm_genpd_resume_noirq(struct device * dev)873 static int pm_genpd_resume_noirq(struct device *dev)
874 {
875 	struct generic_pm_domain *genpd;
876 
877 	dev_dbg(dev, "%s()\n", __func__);
878 
879 	genpd = dev_to_genpd(dev);
880 	if (IS_ERR(genpd))
881 		return -EINVAL;
882 
883 	if (genpd->suspend_power_off)
884 		return 0;
885 
886 	/*
887 	 * Since all of the "noirq" callbacks are executed sequentially, it is
888 	 * guaranteed that this function will never run twice in parallel for
889 	 * the same PM domain, so it is not necessary to use locking here.
890 	 */
891 	pm_genpd_poweron(genpd);
892 	genpd->suspended_count--;
893 	genpd_start_dev(genpd, dev);
894 
895 	return genpd_resume_early(genpd, dev);
896 }
897 
898 /**
899  * pm_genpd_resume - Resume a device belonging to an I/O power domain.
900  * @dev: Device to resume.
901  *
902  * Resume a device under the assumption that its pm_domain field points to the
903  * domain member of an object of type struct generic_pm_domain representing
904  * a power domain consisting of I/O devices.
905  */
pm_genpd_resume(struct device * dev)906 static int pm_genpd_resume(struct device *dev)
907 {
908 	struct generic_pm_domain *genpd;
909 
910 	dev_dbg(dev, "%s()\n", __func__);
911 
912 	genpd = dev_to_genpd(dev);
913 	if (IS_ERR(genpd))
914 		return -EINVAL;
915 
916 	return genpd->suspend_power_off ? 0 : genpd_resume_dev(genpd, dev);
917 }
918 
919 /**
920  * pm_genpd_freeze - Freeze a device belonging to an I/O power domain.
921  * @dev: Device to freeze.
922  *
923  * Freeze a device under the assumption that its pm_domain field points to the
924  * domain member of an object of type struct generic_pm_domain representing
925  * a power domain consisting of I/O devices.
926  */
pm_genpd_freeze(struct device * dev)927 static int pm_genpd_freeze(struct device *dev)
928 {
929 	struct generic_pm_domain *genpd;
930 
931 	dev_dbg(dev, "%s()\n", __func__);
932 
933 	genpd = dev_to_genpd(dev);
934 	if (IS_ERR(genpd))
935 		return -EINVAL;
936 
937 	return genpd->suspend_power_off ? 0 : genpd_freeze_dev(genpd, dev);
938 }
939 
940 /**
941  * pm_genpd_freeze_noirq - Late freeze of a device from an I/O power domain.
942  * @dev: Device to freeze.
943  *
944  * Carry out a late freeze of a device under the assumption that its
945  * pm_domain field points to the domain member of an object of type
946  * struct generic_pm_domain representing a power domain consisting of I/O
947  * devices.
948  */
pm_genpd_freeze_noirq(struct device * dev)949 static int pm_genpd_freeze_noirq(struct device *dev)
950 {
951 	struct generic_pm_domain *genpd;
952 	int ret;
953 
954 	dev_dbg(dev, "%s()\n", __func__);
955 
956 	genpd = dev_to_genpd(dev);
957 	if (IS_ERR(genpd))
958 		return -EINVAL;
959 
960 	if (genpd->suspend_power_off)
961 		return 0;
962 
963 	ret = genpd_freeze_late(genpd, dev);
964 	if (ret)
965 		return ret;
966 
967 	genpd_stop_dev(genpd, dev);
968 
969 	return 0;
970 }
971 
972 /**
973  * pm_genpd_thaw_noirq - Early thaw of a device from an I/O power domain.
974  * @dev: Device to thaw.
975  *
976  * Carry out an early thaw of a device under the assumption that its
977  * pm_domain field points to the domain member of an object of type
978  * struct generic_pm_domain representing a power domain consisting of I/O
979  * devices.
980  */
pm_genpd_thaw_noirq(struct device * dev)981 static int pm_genpd_thaw_noirq(struct device *dev)
982 {
983 	struct generic_pm_domain *genpd;
984 
985 	dev_dbg(dev, "%s()\n", __func__);
986 
987 	genpd = dev_to_genpd(dev);
988 	if (IS_ERR(genpd))
989 		return -EINVAL;
990 
991 	if (genpd->suspend_power_off)
992 		return 0;
993 
994 	genpd_start_dev(genpd, dev);
995 
996 	return genpd_thaw_early(genpd, dev);
997 }
998 
999 /**
1000  * pm_genpd_thaw - Thaw a device belonging to an I/O power domain.
1001  * @dev: Device to thaw.
1002  *
1003  * Thaw a device under the assumption that its pm_domain field points to the
1004  * domain member of an object of type struct generic_pm_domain representing
1005  * a power domain consisting of I/O devices.
1006  */
pm_genpd_thaw(struct device * dev)1007 static int pm_genpd_thaw(struct device *dev)
1008 {
1009 	struct generic_pm_domain *genpd;
1010 
1011 	dev_dbg(dev, "%s()\n", __func__);
1012 
1013 	genpd = dev_to_genpd(dev);
1014 	if (IS_ERR(genpd))
1015 		return -EINVAL;
1016 
1017 	return genpd->suspend_power_off ? 0 : genpd_thaw_dev(genpd, dev);
1018 }
1019 
1020 /**
1021  * pm_genpd_restore_noirq - Early restore of a device from an I/O power domain.
1022  * @dev: Device to resume.
1023  *
1024  * Carry out an early restore of a device under the assumption that its
1025  * pm_domain field points to the domain member of an object of type
1026  * struct generic_pm_domain representing a power domain consisting of I/O
1027  * devices.
1028  */
pm_genpd_restore_noirq(struct device * dev)1029 static int pm_genpd_restore_noirq(struct device *dev)
1030 {
1031 	struct generic_pm_domain *genpd;
1032 
1033 	dev_dbg(dev, "%s()\n", __func__);
1034 
1035 	genpd = dev_to_genpd(dev);
1036 	if (IS_ERR(genpd))
1037 		return -EINVAL;
1038 
1039 	/*
1040 	 * Since all of the "noirq" callbacks are executed sequentially, it is
1041 	 * guaranteed that this function will never run twice in parallel for
1042 	 * the same PM domain, so it is not necessary to use locking here.
1043 	 */
1044 	genpd->status = GPD_STATE_POWER_OFF;
1045 	if (genpd->suspend_power_off) {
1046 		/*
1047 		 * The boot kernel might put the domain into the power on state,
1048 		 * so make sure it really is powered off.
1049 		 */
1050 		if (genpd->power_off)
1051 			genpd->power_off(genpd);
1052 		return 0;
1053 	}
1054 
1055 	pm_genpd_poweron(genpd);
1056 	genpd->suspended_count--;
1057 	genpd_start_dev(genpd, dev);
1058 
1059 	return genpd_resume_early(genpd, dev);
1060 }
1061 
1062 /**
1063  * pm_genpd_complete - Complete power transition of a device in a power domain.
1064  * @dev: Device to complete the transition of.
1065  *
1066  * Complete a power transition of a device (during a system-wide power
1067  * transition) under the assumption that its pm_domain field points to the
1068  * domain member of an object of type struct generic_pm_domain representing
1069  * a power domain consisting of I/O devices.
1070  */
pm_genpd_complete(struct device * dev)1071 static void pm_genpd_complete(struct device *dev)
1072 {
1073 	struct generic_pm_domain *genpd;
1074 	bool run_complete;
1075 
1076 	dev_dbg(dev, "%s()\n", __func__);
1077 
1078 	genpd = dev_to_genpd(dev);
1079 	if (IS_ERR(genpd))
1080 		return;
1081 
1082 	mutex_lock(&genpd->lock);
1083 
1084 	run_complete = !genpd->suspend_power_off;
1085 	if (--genpd->prepared_count == 0)
1086 		genpd->suspend_power_off = false;
1087 
1088 	mutex_unlock(&genpd->lock);
1089 
1090 	if (run_complete) {
1091 		pm_generic_complete(dev);
1092 		pm_runtime_set_active(dev);
1093 		pm_runtime_enable(dev);
1094 		pm_runtime_idle(dev);
1095 	}
1096 }
1097 
1098 #else
1099 
1100 #define pm_genpd_prepare		NULL
1101 #define pm_genpd_suspend		NULL
1102 #define pm_genpd_suspend_noirq		NULL
1103 #define pm_genpd_resume_noirq		NULL
1104 #define pm_genpd_resume			NULL
1105 #define pm_genpd_freeze			NULL
1106 #define pm_genpd_freeze_noirq		NULL
1107 #define pm_genpd_thaw_noirq		NULL
1108 #define pm_genpd_thaw			NULL
1109 #define pm_genpd_restore_noirq		NULL
1110 #define pm_genpd_complete		NULL
1111 
1112 #endif /* CONFIG_PM_SLEEP */
1113 
1114 /**
1115  * __pm_genpd_add_device - Add a device to an I/O PM domain.
1116  * @genpd: PM domain to add the device to.
1117  * @dev: Device to be added.
1118  * @td: Set of PM QoS timing parameters to attach to the device.
1119  */
__pm_genpd_add_device(struct generic_pm_domain * genpd,struct device * dev,struct gpd_timing_data * td)1120 int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
1121 			  struct gpd_timing_data *td)
1122 {
1123 	struct generic_pm_domain_data *gpd_data;
1124 	struct pm_domain_data *pdd;
1125 	int ret = 0;
1126 
1127 	dev_dbg(dev, "%s()\n", __func__);
1128 
1129 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev))
1130 		return -EINVAL;
1131 
1132 	genpd_acquire_lock(genpd);
1133 
1134 	if (genpd->status == GPD_STATE_POWER_OFF) {
1135 		ret = -EINVAL;
1136 		goto out;
1137 	}
1138 
1139 	if (genpd->prepared_count > 0) {
1140 		ret = -EAGAIN;
1141 		goto out;
1142 	}
1143 
1144 	list_for_each_entry(pdd, &genpd->dev_list, list_node)
1145 		if (pdd->dev == dev) {
1146 			ret = -EINVAL;
1147 			goto out;
1148 		}
1149 
1150 	gpd_data = kzalloc(sizeof(*gpd_data), GFP_KERNEL);
1151 	if (!gpd_data) {
1152 		ret = -ENOMEM;
1153 		goto out;
1154 	}
1155 
1156 	genpd->device_count++;
1157 
1158 	dev->pm_domain = &genpd->domain;
1159 	dev_pm_get_subsys_data(dev);
1160 	dev->power.subsys_data->domain_data = &gpd_data->base;
1161 	gpd_data->base.dev = dev;
1162 	gpd_data->need_restore = false;
1163 	list_add_tail(&gpd_data->base.list_node, &genpd->dev_list);
1164 	if (td)
1165 		gpd_data->td = *td;
1166 
1167  out:
1168 	genpd_release_lock(genpd);
1169 
1170 	return ret;
1171 }
1172 
1173 /**
1174  * pm_genpd_remove_device - Remove a device from an I/O PM domain.
1175  * @genpd: PM domain to remove the device from.
1176  * @dev: Device to be removed.
1177  */
pm_genpd_remove_device(struct generic_pm_domain * genpd,struct device * dev)1178 int pm_genpd_remove_device(struct generic_pm_domain *genpd,
1179 			   struct device *dev)
1180 {
1181 	struct pm_domain_data *pdd;
1182 	int ret = -EINVAL;
1183 
1184 	dev_dbg(dev, "%s()\n", __func__);
1185 
1186 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev))
1187 		return -EINVAL;
1188 
1189 	genpd_acquire_lock(genpd);
1190 
1191 	if (genpd->prepared_count > 0) {
1192 		ret = -EAGAIN;
1193 		goto out;
1194 	}
1195 
1196 	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
1197 		if (pdd->dev != dev)
1198 			continue;
1199 
1200 		list_del_init(&pdd->list_node);
1201 		pdd->dev = NULL;
1202 		dev_pm_put_subsys_data(dev);
1203 		dev->pm_domain = NULL;
1204 		kfree(to_gpd_data(pdd));
1205 
1206 		genpd->device_count--;
1207 
1208 		ret = 0;
1209 		break;
1210 	}
1211 
1212  out:
1213 	genpd_release_lock(genpd);
1214 
1215 	return ret;
1216 }
1217 
1218 /**
1219  * pm_genpd_add_subdomain - Add a subdomain to an I/O PM domain.
1220  * @genpd: Master PM domain to add the subdomain to.
1221  * @subdomain: Subdomain to be added.
1222  */
pm_genpd_add_subdomain(struct generic_pm_domain * genpd,struct generic_pm_domain * subdomain)1223 int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
1224 			   struct generic_pm_domain *subdomain)
1225 {
1226 	struct gpd_link *link;
1227 	int ret = 0;
1228 
1229 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain))
1230 		return -EINVAL;
1231 
1232  start:
1233 	genpd_acquire_lock(genpd);
1234 	mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING);
1235 
1236 	if (subdomain->status != GPD_STATE_POWER_OFF
1237 	    && subdomain->status != GPD_STATE_ACTIVE) {
1238 		mutex_unlock(&subdomain->lock);
1239 		genpd_release_lock(genpd);
1240 		goto start;
1241 	}
1242 
1243 	if (genpd->status == GPD_STATE_POWER_OFF
1244 	    &&  subdomain->status != GPD_STATE_POWER_OFF) {
1245 		ret = -EINVAL;
1246 		goto out;
1247 	}
1248 
1249 	list_for_each_entry(link, &genpd->slave_links, slave_node) {
1250 		if (link->slave == subdomain && link->master == genpd) {
1251 			ret = -EINVAL;
1252 			goto out;
1253 		}
1254 	}
1255 
1256 	link = kzalloc(sizeof(*link), GFP_KERNEL);
1257 	if (!link) {
1258 		ret = -ENOMEM;
1259 		goto out;
1260 	}
1261 	link->master = genpd;
1262 	list_add_tail(&link->master_node, &genpd->master_links);
1263 	link->slave = subdomain;
1264 	list_add_tail(&link->slave_node, &subdomain->slave_links);
1265 	if (subdomain->status != GPD_STATE_POWER_OFF)
1266 		genpd_sd_counter_inc(genpd);
1267 
1268  out:
1269 	mutex_unlock(&subdomain->lock);
1270 	genpd_release_lock(genpd);
1271 
1272 	return ret;
1273 }
1274 
1275 /**
1276  * pm_genpd_remove_subdomain - Remove a subdomain from an I/O PM domain.
1277  * @genpd: Master PM domain to remove the subdomain from.
1278  * @subdomain: Subdomain to be removed.
1279  */
pm_genpd_remove_subdomain(struct generic_pm_domain * genpd,struct generic_pm_domain * subdomain)1280 int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
1281 			      struct generic_pm_domain *subdomain)
1282 {
1283 	struct gpd_link *link;
1284 	int ret = -EINVAL;
1285 
1286 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain))
1287 		return -EINVAL;
1288 
1289  start:
1290 	genpd_acquire_lock(genpd);
1291 
1292 	list_for_each_entry(link, &genpd->master_links, master_node) {
1293 		if (link->slave != subdomain)
1294 			continue;
1295 
1296 		mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING);
1297 
1298 		if (subdomain->status != GPD_STATE_POWER_OFF
1299 		    && subdomain->status != GPD_STATE_ACTIVE) {
1300 			mutex_unlock(&subdomain->lock);
1301 			genpd_release_lock(genpd);
1302 			goto start;
1303 		}
1304 
1305 		list_del(&link->master_node);
1306 		list_del(&link->slave_node);
1307 		kfree(link);
1308 		if (subdomain->status != GPD_STATE_POWER_OFF)
1309 			genpd_sd_counter_dec(genpd);
1310 
1311 		mutex_unlock(&subdomain->lock);
1312 
1313 		ret = 0;
1314 		break;
1315 	}
1316 
1317 	genpd_release_lock(genpd);
1318 
1319 	return ret;
1320 }
1321 
1322 /**
1323  * pm_genpd_add_callbacks - Add PM domain callbacks to a given device.
1324  * @dev: Device to add the callbacks to.
1325  * @ops: Set of callbacks to add.
1326  * @td: Timing data to add to the device along with the callbacks (optional).
1327  */
pm_genpd_add_callbacks(struct device * dev,struct gpd_dev_ops * ops,struct gpd_timing_data * td)1328 int pm_genpd_add_callbacks(struct device *dev, struct gpd_dev_ops *ops,
1329 			   struct gpd_timing_data *td)
1330 {
1331 	struct pm_domain_data *pdd;
1332 	int ret = 0;
1333 
1334 	if (!(dev && dev->power.subsys_data && ops))
1335 		return -EINVAL;
1336 
1337 	pm_runtime_disable(dev);
1338 	device_pm_lock();
1339 
1340 	pdd = dev->power.subsys_data->domain_data;
1341 	if (pdd) {
1342 		struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd);
1343 
1344 		gpd_data->ops = *ops;
1345 		if (td)
1346 			gpd_data->td = *td;
1347 	} else {
1348 		ret = -EINVAL;
1349 	}
1350 
1351 	device_pm_unlock();
1352 	pm_runtime_enable(dev);
1353 
1354 	return ret;
1355 }
1356 EXPORT_SYMBOL_GPL(pm_genpd_add_callbacks);
1357 
1358 /**
1359  * __pm_genpd_remove_callbacks - Remove PM domain callbacks from a given device.
1360  * @dev: Device to remove the callbacks from.
1361  * @clear_td: If set, clear the device's timing data too.
1362  */
__pm_genpd_remove_callbacks(struct device * dev,bool clear_td)1363 int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td)
1364 {
1365 	struct pm_domain_data *pdd;
1366 	int ret = 0;
1367 
1368 	if (!(dev && dev->power.subsys_data))
1369 		return -EINVAL;
1370 
1371 	pm_runtime_disable(dev);
1372 	device_pm_lock();
1373 
1374 	pdd = dev->power.subsys_data->domain_data;
1375 	if (pdd) {
1376 		struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd);
1377 
1378 		gpd_data->ops = (struct gpd_dev_ops){ 0 };
1379 		if (clear_td)
1380 			gpd_data->td = (struct gpd_timing_data){ 0 };
1381 	} else {
1382 		ret = -EINVAL;
1383 	}
1384 
1385 	device_pm_unlock();
1386 	pm_runtime_enable(dev);
1387 
1388 	return ret;
1389 }
1390 EXPORT_SYMBOL_GPL(__pm_genpd_remove_callbacks);
1391 
1392 /* Default device callbacks for generic PM domains. */
1393 
1394 /**
1395  * pm_genpd_default_save_state - Default "save device state" for PM domians.
1396  * @dev: Device to handle.
1397  */
pm_genpd_default_save_state(struct device * dev)1398 static int pm_genpd_default_save_state(struct device *dev)
1399 {
1400 	int (*cb)(struct device *__dev);
1401 	struct device_driver *drv = dev->driver;
1402 
1403 	cb = dev_gpd_data(dev)->ops.save_state;
1404 	if (cb)
1405 		return cb(dev);
1406 
1407 	if (drv && drv->pm && drv->pm->runtime_suspend)
1408 		return drv->pm->runtime_suspend(dev);
1409 
1410 	return 0;
1411 }
1412 
1413 /**
1414  * pm_genpd_default_restore_state - Default PM domians "restore device state".
1415  * @dev: Device to handle.
1416  */
pm_genpd_default_restore_state(struct device * dev)1417 static int pm_genpd_default_restore_state(struct device *dev)
1418 {
1419 	int (*cb)(struct device *__dev);
1420 	struct device_driver *drv = dev->driver;
1421 
1422 	cb = dev_gpd_data(dev)->ops.restore_state;
1423 	if (cb)
1424 		return cb(dev);
1425 
1426 	if (drv && drv->pm && drv->pm->runtime_resume)
1427 		return drv->pm->runtime_resume(dev);
1428 
1429 	return 0;
1430 }
1431 
1432 #ifdef CONFIG_PM_SLEEP
1433 
1434 /**
1435  * pm_genpd_default_suspend - Default "device suspend" for PM domians.
1436  * @dev: Device to handle.
1437  */
pm_genpd_default_suspend(struct device * dev)1438 static int pm_genpd_default_suspend(struct device *dev)
1439 {
1440 	int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.suspend;
1441 
1442 	return cb ? cb(dev) : pm_generic_suspend(dev);
1443 }
1444 
1445 /**
1446  * pm_genpd_default_suspend_late - Default "late device suspend" for PM domians.
1447  * @dev: Device to handle.
1448  */
pm_genpd_default_suspend_late(struct device * dev)1449 static int pm_genpd_default_suspend_late(struct device *dev)
1450 {
1451 	int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.suspend_late;
1452 
1453 	return cb ? cb(dev) : pm_generic_suspend_noirq(dev);
1454 }
1455 
1456 /**
1457  * pm_genpd_default_resume_early - Default "early device resume" for PM domians.
1458  * @dev: Device to handle.
1459  */
pm_genpd_default_resume_early(struct device * dev)1460 static int pm_genpd_default_resume_early(struct device *dev)
1461 {
1462 	int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.resume_early;
1463 
1464 	return cb ? cb(dev) : pm_generic_resume_noirq(dev);
1465 }
1466 
1467 /**
1468  * pm_genpd_default_resume - Default "device resume" for PM domians.
1469  * @dev: Device to handle.
1470  */
pm_genpd_default_resume(struct device * dev)1471 static int pm_genpd_default_resume(struct device *dev)
1472 {
1473 	int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.resume;
1474 
1475 	return cb ? cb(dev) : pm_generic_resume(dev);
1476 }
1477 
1478 /**
1479  * pm_genpd_default_freeze - Default "device freeze" for PM domians.
1480  * @dev: Device to handle.
1481  */
pm_genpd_default_freeze(struct device * dev)1482 static int pm_genpd_default_freeze(struct device *dev)
1483 {
1484 	int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.freeze;
1485 
1486 	return cb ? cb(dev) : pm_generic_freeze(dev);
1487 }
1488 
1489 /**
1490  * pm_genpd_default_freeze_late - Default "late device freeze" for PM domians.
1491  * @dev: Device to handle.
1492  */
pm_genpd_default_freeze_late(struct device * dev)1493 static int pm_genpd_default_freeze_late(struct device *dev)
1494 {
1495 	int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.freeze_late;
1496 
1497 	return cb ? cb(dev) : pm_generic_freeze_noirq(dev);
1498 }
1499 
1500 /**
1501  * pm_genpd_default_thaw_early - Default "early device thaw" for PM domians.
1502  * @dev: Device to handle.
1503  */
pm_genpd_default_thaw_early(struct device * dev)1504 static int pm_genpd_default_thaw_early(struct device *dev)
1505 {
1506 	int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.thaw_early;
1507 
1508 	return cb ? cb(dev) : pm_generic_thaw_noirq(dev);
1509 }
1510 
1511 /**
1512  * pm_genpd_default_thaw - Default "device thaw" for PM domians.
1513  * @dev: Device to handle.
1514  */
pm_genpd_default_thaw(struct device * dev)1515 static int pm_genpd_default_thaw(struct device *dev)
1516 {
1517 	int (*cb)(struct device *__dev) = dev_gpd_data(dev)->ops.thaw;
1518 
1519 	return cb ? cb(dev) : pm_generic_thaw(dev);
1520 }
1521 
1522 #else /* !CONFIG_PM_SLEEP */
1523 
1524 #define pm_genpd_default_suspend	NULL
1525 #define pm_genpd_default_suspend_late	NULL
1526 #define pm_genpd_default_resume_early	NULL
1527 #define pm_genpd_default_resume		NULL
1528 #define pm_genpd_default_freeze		NULL
1529 #define pm_genpd_default_freeze_late	NULL
1530 #define pm_genpd_default_thaw_early	NULL
1531 #define pm_genpd_default_thaw		NULL
1532 
1533 #endif /* !CONFIG_PM_SLEEP */
1534 
1535 /**
1536  * pm_genpd_init - Initialize a generic I/O PM domain object.
1537  * @genpd: PM domain object to initialize.
1538  * @gov: PM domain governor to associate with the domain (may be NULL).
1539  * @is_off: Initial value of the domain's power_is_off field.
1540  */
pm_genpd_init(struct generic_pm_domain * genpd,struct dev_power_governor * gov,bool is_off)1541 void pm_genpd_init(struct generic_pm_domain *genpd,
1542 		   struct dev_power_governor *gov, bool is_off)
1543 {
1544 	if (IS_ERR_OR_NULL(genpd))
1545 		return;
1546 
1547 	INIT_LIST_HEAD(&genpd->master_links);
1548 	INIT_LIST_HEAD(&genpd->slave_links);
1549 	INIT_LIST_HEAD(&genpd->dev_list);
1550 	mutex_init(&genpd->lock);
1551 	genpd->gov = gov;
1552 	INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn);
1553 	genpd->in_progress = 0;
1554 	atomic_set(&genpd->sd_count, 0);
1555 	genpd->status = is_off ? GPD_STATE_POWER_OFF : GPD_STATE_ACTIVE;
1556 	init_waitqueue_head(&genpd->status_wait_queue);
1557 	genpd->poweroff_task = NULL;
1558 	genpd->resume_count = 0;
1559 	genpd->device_count = 0;
1560 	genpd->suspended_count = 0;
1561 	genpd->max_off_time_ns = -1;
1562 	genpd->domain.ops.runtime_suspend = pm_genpd_runtime_suspend;
1563 	genpd->domain.ops.runtime_resume = pm_genpd_runtime_resume;
1564 	genpd->domain.ops.runtime_idle = pm_generic_runtime_idle;
1565 	genpd->domain.ops.prepare = pm_genpd_prepare;
1566 	genpd->domain.ops.suspend = pm_genpd_suspend;
1567 	genpd->domain.ops.suspend_noirq = pm_genpd_suspend_noirq;
1568 	genpd->domain.ops.resume_noirq = pm_genpd_resume_noirq;
1569 	genpd->domain.ops.resume = pm_genpd_resume;
1570 	genpd->domain.ops.freeze = pm_genpd_freeze;
1571 	genpd->domain.ops.freeze_noirq = pm_genpd_freeze_noirq;
1572 	genpd->domain.ops.thaw_noirq = pm_genpd_thaw_noirq;
1573 	genpd->domain.ops.thaw = pm_genpd_thaw;
1574 	genpd->domain.ops.poweroff = pm_genpd_suspend;
1575 	genpd->domain.ops.poweroff_noirq = pm_genpd_suspend_noirq;
1576 	genpd->domain.ops.restore_noirq = pm_genpd_restore_noirq;
1577 	genpd->domain.ops.restore = pm_genpd_resume;
1578 	genpd->domain.ops.complete = pm_genpd_complete;
1579 	genpd->dev_ops.save_state = pm_genpd_default_save_state;
1580 	genpd->dev_ops.restore_state = pm_genpd_default_restore_state;
1581 	genpd->dev_ops.suspend = pm_genpd_default_suspend;
1582 	genpd->dev_ops.suspend_late = pm_genpd_default_suspend_late;
1583 	genpd->dev_ops.resume_early = pm_genpd_default_resume_early;
1584 	genpd->dev_ops.resume = pm_genpd_default_resume;
1585 	genpd->dev_ops.freeze = pm_genpd_default_freeze;
1586 	genpd->dev_ops.freeze_late = pm_genpd_default_freeze_late;
1587 	genpd->dev_ops.thaw_early = pm_genpd_default_thaw_early;
1588 	genpd->dev_ops.thaw = pm_genpd_default_thaw;
1589 	mutex_lock(&gpd_list_lock);
1590 	list_add(&genpd->gpd_list_node, &gpd_list);
1591 	mutex_unlock(&gpd_list_lock);
1592 }
1593