1 // SPDX-License-Identifier: GPL-2.0-only
19  *   - The 'prio' field contains the priority of the context that owns the
20  *     console. Only higher priority contexts are allowed to take over the
23  *   - The 'cpu' field denotes on which CPU the console is locked. It is used
26  *     taken over by a higher priority context, released, and taken on another
31  *   - The 'req_prio' field is used by the handover approach to make the
32  *     current owner aware that there is a context with a higher priority
35  *   - The 'unsafe' field allows to take over the console in a safe way in the
41  *   - The 'unsafe_takeover' field is set when a hostile takeover took the
43  *     until re-initialized.
51  *      when the current owner has lower priority and the console is in an
70  *   3) Unsafe hostile takeover allows to take over the lock even when the
85  *   - Preference for higher priority contexts.
86  *   - Protection of the panic CPU.
90  *   - What is marked as an unsafe section.
91  *   - Whether to spin-wait if there is already an owner and the console is
93  *   - Whether to attempt an unsafe hostile takeover.
107  * nbcon_state_set - Helper function to set the console state
116 	atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom);  in nbcon_state_set()
120  * nbcon_state_read - Helper function to read the console state
126 	state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state));  in nbcon_state_read()
130  * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state
140 	return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom);  in nbcon_state_try_cmpxchg()
169 	seq = rb_next_seq - ((u32)rb_next_seq - nbcon_seq);  in __nbcon_seq_to_seq()
177  * nbcon_seq_read - Read the current console sequence
190  * nbcon_seq_force - Force console sequence to a specific value
209 	/* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */  in nbcon_seq_force()
210 	con->seq = 0;  in nbcon_seq_force()
214  * nbcon_seq_try_update - Try to update the console sequence number
219  * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to
221  * nbcon_seq_force() was used or the current context no longer owns the
226 	unsigned long nbcon_seq = __seq_to_nbcon_seq(ctxt->seq);  in nbcon_seq_try_update()
227 	struct console *con = ctxt->console;  in nbcon_seq_try_update()
231 		ctxt->seq = new_seq;  in nbcon_seq_try_update()
233 		ctxt->seq = nbcon_seq_read(con);  in nbcon_seq_try_update()
238  * nbcon_context_try_acquire_direct - Try to acquire directly
240  * @cur:	The current console state
243  * the current owner has a lower priority and the console is in a safe state.
250  *	-EPERM:		A panic is in progress and this is not the panic CPU.
251  *			Or the current owner or waiter has the same or higher
255  *	-EBUSY:		The current owner has a lower priority but the console
263 	struct console *con = ctxt->console;  in nbcon_context_try_acquire_direct()
268 			return -EPERM;  in nbcon_context_try_acquire_direct()
270 		if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio)  in nbcon_context_try_acquire_direct()
271 			return -EPERM;  in nbcon_context_try_acquire_direct()
273 		if (cur->unsafe)  in nbcon_context_try_acquire_direct()
274 			return -EBUSY;  in nbcon_context_try_acquire_direct()
280 		WARN_ON_ONCE(cur->unsafe_takeover);  in nbcon_context_try_acquire_direct()
282 		new.atom = cur->atom;  in nbcon_context_try_acquire_direct()
283 		new.prio	= ctxt->prio;  in nbcon_context_try_acquire_direct()
285 		new.unsafe	= cur->unsafe_takeover;  in nbcon_context_try_acquire_direct()
298 	 * - Only a context with a higher priority can take over the request.  in nbcon_waiter_matches()
299 	 * - There are only three priorities.  in nbcon_waiter_matches()
300 	 * - Only one CPU is allowed to request PANIC priority.  in nbcon_waiter_matches()
301 	 * - Lower priorities are ignored during panic() until reboot.  in nbcon_waiter_matches()
312 	return (cur->req_prio == expected_prio);  in nbcon_waiter_matches()
316  * nbcon_context_try_acquire_requested - Try to acquire after having
319  * @cur:	The current console state
322  * It is called when the console is in an unsafe state. The current
330  *	-EPERM:		A panic is in progress and this is not the panic CPU
333  *	-EBUSY:		The console is still locked. The caller should
343 	struct console *con = ctxt->console;  in nbcon_context_try_acquire_requested()
348 		return -EPERM;  in nbcon_context_try_acquire_requested()
354 	if (!nbcon_waiter_matches(cur, ctxt->prio))  in nbcon_context_try_acquire_requested()
355 		return -EPERM;  in nbcon_context_try_acquire_requested()
358 	if (cur->prio != NBCON_PRIO_NONE)  in nbcon_context_try_acquire_requested()
359 		return -EBUSY;  in nbcon_context_try_acquire_requested()
365 	WARN_ON_ONCE(cur->unsafe);  in nbcon_context_try_acquire_requested()
367 	new.atom = cur->atom;  in nbcon_context_try_acquire_requested()
368 	new.prio	= ctxt->prio;  in nbcon_context_try_acquire_requested()
370 	new.unsafe	= cur->unsafe_takeover;  in nbcon_context_try_acquire_requested()
376 		 * over by a higher priority context.  in nbcon_context_try_acquire_requested()
378 		WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio));  in nbcon_context_try_acquire_requested()
379 		return -EPERM;  in nbcon_context_try_acquire_requested()
387  * nbcon_context_try_acquire_handover - Try to acquire via handover
389  * @cur:	The current console state
392  * than the current owner and the console is in an unsafe state.
393  * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY.
395  * The function sets "req_prio" field to make the current owner aware of
396  * the request. Then it waits until the current owner releases the console,
397  * or an even higher context takes over the request, or timeout expires.
399  * The current owner checks the "req_prio" field on exit from the unsafe
408  *	-EPERM:		A panic is in progress and this is not the panic CPU.
409  *			Or a higher priority context has taken over the
412  *	-EBUSY:		The current owner is on the same CPU so that the hand
413  *			shake could not work. Or the current owner is not
419  *	-EAGAIN:	@cur has changed when creating the handover request.
426 	struct console *con = ctxt->console;  in nbcon_context_try_acquire_handover()
429 	int request_err = -EBUSY;  in nbcon_context_try_acquire_handover()
433 	 * with -EBUSY.  in nbcon_context_try_acquire_handover()
435 	WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);  in nbcon_context_try_acquire_handover()
436 	WARN_ON_ONCE(!cur->unsafe);  in nbcon_context_try_acquire_handover()
439 	if (cur->cpu == cpu)  in nbcon_context_try_acquire_handover()
440 		return -EBUSY;  in nbcon_context_try_acquire_handover()
443 	 * Console stays unsafe after an unsafe takeover until re-initialized.  in nbcon_context_try_acquire_handover()
446 	if (cur->unsafe_takeover)  in nbcon_context_try_acquire_handover()
447 		return -EBUSY;  in nbcon_context_try_acquire_handover()
450 	if (ctxt->spinwait_max_us == 0)  in nbcon_context_try_acquire_handover()
451 		return -EBUSY;  in nbcon_context_try_acquire_handover()
455 	 * the console directly when the current state has been modified.  in nbcon_context_try_acquire_handover()
457 	new.atom = cur->atom;  in nbcon_context_try_acquire_handover()
458 	new.req_prio = ctxt->prio;  in nbcon_context_try_acquire_handover()
460 		return -EAGAIN;  in nbcon_context_try_acquire_handover()
462 	cur->atom = new.atom;  in nbcon_context_try_acquire_handover()
465 	for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) {  in nbcon_context_try_acquire_handover()
475 		if (request_err == -EPERM)  in nbcon_context_try_acquire_handover()
480 		/* Re-read the state because some time has passed. */  in nbcon_context_try_acquire_handover()
488 		 * can only happen if a higher priority context has taken over  in nbcon_context_try_acquire_handover()
491 		if (!nbcon_waiter_matches(cur, ctxt->prio))  in nbcon_context_try_acquire_handover()
492 			return -EPERM;  in nbcon_context_try_acquire_handover()
495 		new.atom = cur->atom;  in nbcon_context_try_acquire_handover()
502 			cur->atom = new.atom;  in nbcon_context_try_acquire_handover()
517  * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover
519  * @cur:	The current console state
526  * Return:	0 on success. -EPERM when not allowed by the context.
532 	struct console *con = ctxt->console;  in nbcon_context_try_acquire_hostile()
535 	if (!ctxt->allow_unsafe_takeover)  in nbcon_context_try_acquire_hostile()
536 		return -EPERM;  in nbcon_context_try_acquire_hostile()
539 	if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC))  in nbcon_context_try_acquire_hostile()
540 		return -EPERM;  in nbcon_context_try_acquire_hostile()
544 	 * -EBUSY in the right situation.  in nbcon_context_try_acquire_hostile()
546 	WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);  in nbcon_context_try_acquire_hostile()
547 	WARN_ON_ONCE(cur->unsafe != true);  in nbcon_context_try_acquire_hostile()
550 		new.atom = cur->atom;  in nbcon_context_try_acquire_hostile()
552 		new.prio		= ctxt->prio;  in nbcon_context_try_acquire_hostile()
553 		new.unsafe		|= cur->unsafe_takeover;  in nbcon_context_try_acquire_hostile()
554 		new.unsafe_takeover	|= cur->unsafe;  in nbcon_context_try_acquire_hostile()
564  * nbcon_context_try_acquire - Try to acquire nbcon console
570  * caller should check the current console state to see if it is
578 	struct console *con = ctxt->console;  in nbcon_context_try_acquire()
585 	if (err != -EBUSY)  in nbcon_context_try_acquire()
589 	if (err == -EAGAIN)  in nbcon_context_try_acquire()
591 	if (err != -EBUSY)  in nbcon_context_try_acquire()
603 		ctxt->pbufs = &panic_nbcon_pbufs;  in nbcon_context_try_acquire()
605 		ctxt->pbufs = con->pbufs;  in nbcon_context_try_acquire()
608 	ctxt->seq = nbcon_seq_read(ctxt->console);  in nbcon_context_try_acquire()
624 	if (cur->prio != expected_prio)  in nbcon_owner_matches()
627 	if (cur->cpu != expected_cpu)  in nbcon_owner_matches()
634  * nbcon_context_release - Release the console
640 	struct console *con = ctxt->console;  in nbcon_context_release()
647 		if (!nbcon_owner_matches(&cur, cpu, ctxt->prio))  in nbcon_context_release()
661 	ctxt->pbufs = NULL;  in nbcon_context_release()
665  * nbcon_context_can_proceed - Check whether ownership can proceed
667  * @cur:	The current console state
670  *		ownership was handed over or taken.
695 	if (!nbcon_owner_matches(cur, cpu, ctxt->prio))  in nbcon_context_can_proceed()
699 	if (cur->req_prio == NBCON_PRIO_NONE)  in nbcon_context_can_proceed()
708 	if (cur->unsafe)  in nbcon_context_can_proceed()
712 	WARN_ON_ONCE(cur->req_prio <= cur->prio);  in nbcon_context_can_proceed()
715 	 * Having a safe point for take over and eventually a few  in nbcon_context_can_proceed()
718 	 * Release and hand over.  in nbcon_context_can_proceed()
723 	 * It is not clear whether the waiter really took over ownership. The  in nbcon_context_can_proceed()
736  * nbcon_can_proceed - Check whether ownership can proceed
740  *		ownership was handed over or taken.
761 	struct console *con = ctxt->console;  in nbcon_can_proceed()
774  * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state
780  *		over or taken.
794 	struct console *con = ctxt->console;  in __nbcon_context_update_unsafe()
821  * nbcon_enter_unsafe - Enter an unsafe region in the driver
825  *		ownership was handed over or taken.
841  * nbcon_exit_unsafe - Exit an unsafe region in the driver
845  *		ownership was handed over or taken.
861  * nbcon_emit_next_record - Emit a record in the acquired context
865  *		ownership was handed over or taken.
873  * When true is returned, @wctxt->ctxt.backlog indicates whether there are
880 	struct console *con = ctxt->console;  in nbcon_emit_next_record()
883 		.pbufs = ctxt->pbufs,  in nbcon_emit_next_record()
899 	ctxt->backlog = printk_get_next_message(&pmsg, ctxt->seq, is_extended, true);  in nbcon_emit_next_record()
900 	if (!ctxt->backlog)  in nbcon_emit_next_record()
904 	 * @con->dropped is not protected in case of an unsafe hostile  in nbcon_emit_next_record()
908 	con_dropped = data_race(READ_ONCE(con->dropped));  in nbcon_emit_next_record()
922 	wctxt->outbuf = &pmsg.pbufs->outbuf[0];  in nbcon_emit_next_record()
923 	wctxt->len = pmsg.outbuf_len;  in nbcon_emit_next_record()
925 	wctxt->unsafe_takeover = cur.unsafe_takeover;  in nbcon_emit_next_record()
927 	if (con->write_atomic) {  in nbcon_emit_next_record()
928 		done = con->write_atomic(con, wctxt);  in nbcon_emit_next_record()
956 		WRITE_ONCE(con->dropped, dropped);  in nbcon_emit_next_record()
965  * nbcon_alloc - Allocate buffers needed by the nbcon console
976 	if (con->flags & CON_BOOT) {  in nbcon_alloc()
982 		con->pbufs = &printk_shared_pbufs;  in nbcon_alloc()
984 		con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL);  in nbcon_alloc()
985 		if (!con->pbufs) {  in nbcon_alloc()
995  * nbcon_init - Initialize the nbcon console specific data
1001  * This function expects that the legacy @con->seq has been set.
1008 	BUG_ON(!con->pbufs);  in nbcon_init()
1010 	nbcon_seq_force(con, con->seq);  in nbcon_init()
1015  * nbcon_free - Free and cleanup the nbcon console specific data
1025 	if (!(con->flags & CON_BOOT))  in nbcon_free()
1026 		kfree(con->pbufs);  in nbcon_free()
1028 	con->pbufs = NULL;  in nbcon_free()