1 // SPDX-License-Identifier: GPL-2.0-only
2
3 /* -----------------------------------------------------------------------
4 *
5 * Copyright 2011 Intel Corporation; author Matt Fleming
6 *
7 * ----------------------------------------------------------------------- */
8
9 #include <linux/efi.h>
10 #include <linux/pci.h>
11 #include <linux/stddef.h>
12
13 #include <asm/efi.h>
14 #include <asm/e820/types.h>
15 #include <asm/setup.h>
16 #include <asm/desc.h>
17 #include <asm/boot.h>
18 #include <asm/kaslr.h>
19 #include <asm/sev.h>
20
21 #include "efistub.h"
22 #include "x86-stub.h"
23
24 extern char _bss[], _ebss[];
25
26 const efi_system_table_t *efi_system_table;
27 const efi_dxe_services_table_t *efi_dxe_table;
28 static efi_loaded_image_t *image = NULL;
29 static efi_memory_attribute_protocol_t *memattr;
30
31 typedef union sev_memory_acceptance_protocol sev_memory_acceptance_protocol_t;
32 union sev_memory_acceptance_protocol {
33 struct {
34 efi_status_t (__efiapi * allow_unaccepted_memory)(
35 sev_memory_acceptance_protocol_t *);
36 };
37 struct {
38 u32 allow_unaccepted_memory;
39 } mixed_mode;
40 };
41
42 static efi_status_t
preserve_pci_rom_image(efi_pci_io_protocol_t * pci,struct pci_setup_rom ** __rom)43 preserve_pci_rom_image(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom)
44 {
45 struct pci_setup_rom *rom __free(efi_pool) = NULL;
46 efi_status_t status;
47 unsigned long size;
48 uint64_t romsize;
49 void *romimage;
50
51 /*
52 * Some firmware images contain EFI function pointers at the place where
53 * the romimage and romsize fields are supposed to be. Typically the EFI
54 * code is mapped at high addresses, translating to an unrealistically
55 * large romsize. The UEFI spec limits the size of option ROMs to 16
56 * MiB so we reject any ROMs over 16 MiB in size to catch this.
57 */
58 romimage = efi_table_attr(pci, romimage);
59 romsize = efi_table_attr(pci, romsize);
60 if (!romimage || !romsize || romsize > SZ_16M)
61 return EFI_INVALID_PARAMETER;
62
63 size = romsize + sizeof(*rom);
64
65 status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size,
66 (void **)&rom);
67 if (status != EFI_SUCCESS) {
68 efi_err("Failed to allocate memory for 'rom'\n");
69 return status;
70 }
71
72 memset(rom, 0, sizeof(*rom));
73
74 rom->data.type = SETUP_PCI;
75 rom->data.len = size - sizeof(struct setup_data);
76 rom->data.next = 0;
77 rom->pcilen = romsize;
78
79 status = efi_call_proto(pci, pci.read, EfiPciIoWidthUint16,
80 PCI_VENDOR_ID, 1, &rom->vendor);
81
82 if (status != EFI_SUCCESS) {
83 efi_err("Failed to read rom->vendor\n");
84 return status;
85 }
86
87 status = efi_call_proto(pci, pci.read, EfiPciIoWidthUint16,
88 PCI_DEVICE_ID, 1, &rom->devid);
89
90 if (status != EFI_SUCCESS) {
91 efi_err("Failed to read rom->devid\n");
92 return status;
93 }
94
95 status = efi_call_proto(pci, get_location, &rom->segment, &rom->bus,
96 &rom->device, &rom->function);
97
98 if (status != EFI_SUCCESS)
99 return status;
100
101 memcpy(rom->romdata, romimage, romsize);
102 *__rom = no_free_ptr(rom);
103 return EFI_SUCCESS;
104 }
105
106 /*
107 * There's no way to return an informative status from this function,
108 * because any analysis (and printing of error messages) needs to be
109 * done directly at the EFI function call-site.
110 *
111 * For example, EFI_INVALID_PARAMETER could indicate a bug or maybe we
112 * just didn't find any PCI devices, but there's no way to tell outside
113 * the context of the call.
114 */
setup_efi_pci(struct boot_params * params)115 static void setup_efi_pci(struct boot_params *params)
116 {
117 efi_status_t status;
118 efi_handle_t *pci_handle __free(efi_pool) = NULL;
119 efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID;
120 struct setup_data *data;
121 unsigned long num;
122 efi_handle_t h;
123
124 status = efi_bs_call(locate_handle_buffer, EFI_LOCATE_BY_PROTOCOL,
125 &pci_proto, NULL, &num, &pci_handle);
126 if (status != EFI_SUCCESS)
127 return;
128
129 data = (struct setup_data *)(unsigned long)params->hdr.setup_data;
130
131 while (data && data->next)
132 data = (struct setup_data *)(unsigned long)data->next;
133
134 for_each_efi_handle(h, pci_handle, num) {
135 efi_pci_io_protocol_t *pci = NULL;
136 struct pci_setup_rom *rom;
137
138 status = efi_bs_call(handle_protocol, h, &pci_proto,
139 (void **)&pci);
140 if (status != EFI_SUCCESS || !pci)
141 continue;
142
143 status = preserve_pci_rom_image(pci, &rom);
144 if (status != EFI_SUCCESS)
145 continue;
146
147 if (data)
148 data->next = (unsigned long)rom;
149 else
150 params->hdr.setup_data = (unsigned long)rom;
151
152 data = (struct setup_data *)rom;
153 }
154 }
155
retrieve_apple_device_properties(struct boot_params * boot_params)156 static void retrieve_apple_device_properties(struct boot_params *boot_params)
157 {
158 efi_guid_t guid = APPLE_PROPERTIES_PROTOCOL_GUID;
159 struct setup_data *data, *new;
160 efi_status_t status;
161 u32 size = 0;
162 apple_properties_protocol_t *p;
163
164 status = efi_bs_call(locate_protocol, &guid, NULL, (void **)&p);
165 if (status != EFI_SUCCESS)
166 return;
167
168 if (efi_table_attr(p, version) != 0x10000) {
169 efi_err("Unsupported properties proto version\n");
170 return;
171 }
172
173 efi_call_proto(p, get_all, NULL, &size);
174 if (!size)
175 return;
176
177 do {
178 status = efi_bs_call(allocate_pool, EFI_LOADER_DATA,
179 size + sizeof(struct setup_data),
180 (void **)&new);
181 if (status != EFI_SUCCESS) {
182 efi_err("Failed to allocate memory for 'properties'\n");
183 return;
184 }
185
186 status = efi_call_proto(p, get_all, new->data, &size);
187
188 if (status == EFI_BUFFER_TOO_SMALL)
189 efi_bs_call(free_pool, new);
190 } while (status == EFI_BUFFER_TOO_SMALL);
191
192 new->type = SETUP_APPLE_PROPERTIES;
193 new->len = size;
194 new->next = 0;
195
196 data = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data;
197 if (!data) {
198 boot_params->hdr.setup_data = (unsigned long)new;
199 } else {
200 while (data->next)
201 data = (struct setup_data *)(unsigned long)data->next;
202 data->next = (unsigned long)new;
203 }
204 }
205
206 struct smbios_entry_point {
207 u8 anchor[4];
208 u8 ep_checksum;
209 u8 ep_length;
210 u8 major_version;
211 u8 minor_version;
212 u16 max_size_entry;
213 u8 ep_rev;
214 u8 reserved[5];
215
216 struct __packed {
217 u8 anchor[5];
218 u8 checksum;
219 u16 st_length;
220 u32 st_address;
221 u16 number_of_entries;
222 u8 bcd_rev;
223 } intm;
224 };
225
verify_ep_checksum(const void * ptr,int length)226 static bool verify_ep_checksum(const void *ptr, int length)
227 {
228 u8 sum = 0;
229
230 for (int i = 0; i < length; i++)
231 sum += ((u8 *)ptr)[i];
232
233 return sum == 0;
234 }
235
verify_ep_integrity(const struct smbios_entry_point * ep)236 static bool verify_ep_integrity(const struct smbios_entry_point *ep)
237 {
238 if (memcmp(ep->anchor, "_SM_", sizeof(ep->anchor)) != 0)
239 return false;
240
241 if (memcmp(ep->intm.anchor, "_DMI_", sizeof(ep->intm.anchor)) != 0)
242 return false;
243
244 if (!verify_ep_checksum(ep, ep->ep_length) ||
245 !verify_ep_checksum(&ep->intm, sizeof(ep->intm)))
246 return false;
247
248 return true;
249 }
250
search_record(void * table,u32 length,u8 type)251 static const struct efi_smbios_record *search_record(void *table, u32 length,
252 u8 type)
253 {
254 const u8 *p, *end;
255
256 p = (u8 *)table;
257 end = p + length;
258
259 while (p + sizeof(struct efi_smbios_record) < end) {
260 const struct efi_smbios_record *hdr =
261 (struct efi_smbios_record *)p;
262 const u8 *next;
263
264 if (hdr->type == type)
265 return hdr;
266
267 /* Type 127 = End-of-Table */
268 if (hdr->type == 0x7F)
269 return NULL;
270
271 /* Jumping to the unformed section */
272 next = p + hdr->length;
273
274 /* Unformed section ends with 0000h */
275 while ((next[0] != 0 || next[1] != 0) && next + 1 < end)
276 next++;
277
278 next += 2;
279 p = next;
280 }
281
282 return NULL;
283 }
284
get_table_record(u8 type)285 static const struct efi_smbios_record *get_table_record(u8 type)
286 {
287 const struct smbios_entry_point *ep;
288
289 /*
290 * Locate the legacy 32-bit SMBIOS entrypoint in memory, and parse it
291 * directly. Needed by some Macs that do not implement the EFI protocol.
292 */
293 ep = get_efi_config_table(SMBIOS_TABLE_GUID);
294 if (!ep)
295 return NULL;
296
297 if (!verify_ep_integrity(ep))
298 return NULL;
299
300 return search_record((void *)(unsigned long)ep->intm.st_address,
301 ep->intm.st_length, type);
302 }
303
apple_match_product_name(void)304 static bool apple_match_product_name(void)
305 {
306 static const char type1_product_matches[][15] = {
307 "MacBookPro11,3",
308 "MacBookPro11,5",
309 "MacBookPro13,3",
310 "MacBookPro14,3",
311 "MacBookPro15,1",
312 "MacBookPro15,3",
313 "MacBookPro16,1",
314 "MacBookPro16,4",
315 };
316 const struct efi_smbios_type1_record *record;
317 const u8 *product;
318
319 record = (struct efi_smbios_type1_record *)
320 (efi_get_smbios_record(1) ?: get_table_record(1));
321 if (!record)
322 return false;
323
324 product = efi_get_smbios_string(record, product_name);
325 if (!product)
326 return false;
327
328 for (int i = 0; i < ARRAY_SIZE(type1_product_matches); i++) {
329 if (!strcmp(product, type1_product_matches[i]))
330 return true;
331 }
332
333 return false;
334 }
335
apple_set_os(void)336 static void apple_set_os(void)
337 {
338 struct {
339 unsigned long version;
340 efi_status_t (__efiapi *set_os_version)(const char *);
341 efi_status_t (__efiapi *set_os_vendor)(const char *);
342 } *set_os;
343 efi_status_t status;
344
345 if (!efi_is_64bit() || !apple_match_product_name())
346 return;
347
348 status = efi_bs_call(locate_protocol, &APPLE_SET_OS_PROTOCOL_GUID, NULL,
349 (void **)&set_os);
350 if (status != EFI_SUCCESS)
351 return;
352
353 if (set_os->version >= 2) {
354 status = set_os->set_os_vendor("Apple Inc.");
355 if (status != EFI_SUCCESS)
356 efi_err("Failed to set OS vendor via apple_set_os\n");
357 }
358
359 if (set_os->version > 0) {
360 /* The version being set doesn't seem to matter */
361 status = set_os->set_os_version("Mac OS X 10.9");
362 if (status != EFI_SUCCESS)
363 efi_err("Failed to set OS version via apple_set_os\n");
364 }
365 }
366
efi_adjust_memory_range_protection(unsigned long start,unsigned long size)367 efi_status_t efi_adjust_memory_range_protection(unsigned long start,
368 unsigned long size)
369 {
370 efi_status_t status;
371 efi_gcd_memory_space_desc_t desc;
372 unsigned long end, next;
373 unsigned long rounded_start, rounded_end;
374 unsigned long unprotect_start, unprotect_size;
375
376 rounded_start = rounddown(start, EFI_PAGE_SIZE);
377 rounded_end = roundup(start + size, EFI_PAGE_SIZE);
378
379 if (memattr != NULL) {
380 status = efi_call_proto(memattr, set_memory_attributes,
381 rounded_start,
382 rounded_end - rounded_start,
383 EFI_MEMORY_RO);
384 if (status != EFI_SUCCESS) {
385 efi_warn("Failed to set EFI_MEMORY_RO attribute\n");
386 return status;
387 }
388
389 status = efi_call_proto(memattr, clear_memory_attributes,
390 rounded_start,
391 rounded_end - rounded_start,
392 EFI_MEMORY_XP);
393 if (status != EFI_SUCCESS)
394 efi_warn("Failed to clear EFI_MEMORY_XP attribute\n");
395 return status;
396 }
397
398 if (efi_dxe_table == NULL)
399 return EFI_SUCCESS;
400
401 /*
402 * Don't modify memory region attributes, if they are
403 * already suitable, to lower the possibility to
404 * encounter firmware bugs.
405 */
406
407 for (end = start + size; start < end; start = next) {
408
409 status = efi_dxe_call(get_memory_space_descriptor, start, &desc);
410
411 if (status != EFI_SUCCESS)
412 break;
413
414 next = desc.base_address + desc.length;
415
416 /*
417 * Only system memory and more reliable memory are suitable for
418 * trampoline/kernel image placement. So only those memory types
419 * may need to have attributes modified.
420 */
421
422 if ((desc.gcd_memory_type != EfiGcdMemoryTypeSystemMemory &&
423 desc.gcd_memory_type != EfiGcdMemoryTypeMoreReliable) ||
424 (desc.attributes & (EFI_MEMORY_RO | EFI_MEMORY_XP)) == 0)
425 continue;
426
427 unprotect_start = max(rounded_start, (unsigned long)desc.base_address);
428 unprotect_size = min(rounded_end, next) - unprotect_start;
429
430 status = efi_dxe_call(set_memory_space_attributes,
431 unprotect_start, unprotect_size,
432 EFI_MEMORY_WB);
433
434 if (status != EFI_SUCCESS) {
435 efi_warn("Unable to unprotect memory range [%08lx,%08lx]: %lx\n",
436 unprotect_start,
437 unprotect_start + unprotect_size,
438 status);
439 break;
440 }
441 }
442 return EFI_SUCCESS;
443 }
444
setup_unaccepted_memory(void)445 static void setup_unaccepted_memory(void)
446 {
447 efi_guid_t mem_acceptance_proto = OVMF_SEV_MEMORY_ACCEPTANCE_PROTOCOL_GUID;
448 sev_memory_acceptance_protocol_t *proto;
449 efi_status_t status;
450
451 if (!IS_ENABLED(CONFIG_UNACCEPTED_MEMORY))
452 return;
453
454 /*
455 * Enable unaccepted memory before calling exit boot services in order
456 * for the UEFI to not accept all memory on EBS.
457 */
458 status = efi_bs_call(locate_protocol, &mem_acceptance_proto, NULL,
459 (void **)&proto);
460 if (status != EFI_SUCCESS)
461 return;
462
463 status = efi_call_proto(proto, allow_unaccepted_memory);
464 if (status != EFI_SUCCESS)
465 efi_err("Memory acceptance protocol failed\n");
466 }
467
efistub_fw_vendor(void)468 static efi_char16_t *efistub_fw_vendor(void)
469 {
470 unsigned long vendor = efi_table_attr(efi_system_table, fw_vendor);
471
472 return (efi_char16_t *)vendor;
473 }
474
475 static const efi_char16_t apple[] = L"Apple";
476
setup_quirks(struct boot_params * boot_params)477 static void setup_quirks(struct boot_params *boot_params)
478 {
479 if (!memcmp(efistub_fw_vendor(), apple, sizeof(apple))) {
480 if (IS_ENABLED(CONFIG_APPLE_PROPERTIES))
481 retrieve_apple_device_properties(boot_params);
482
483 apple_set_os();
484 }
485 }
486
setup_graphics(struct boot_params * boot_params)487 static void setup_graphics(struct boot_params *boot_params)
488 {
489 struct screen_info *si = memset(&boot_params->screen_info, 0, sizeof(*si));
490 struct edid_info *edid = memset(&boot_params->edid_info, 0, sizeof(*edid));
491
492 efi_setup_graphics(si, edid);
493 }
494
efi_exit(efi_handle_t handle,efi_status_t status)495 static void __noreturn efi_exit(efi_handle_t handle, efi_status_t status)
496 {
497 efi_bs_call(exit, handle, status, 0, NULL);
498 for(;;)
499 asm("hlt");
500 }
501
502 /*
503 * Because the x86 boot code expects to be passed a boot_params we
504 * need to create one ourselves (usually the bootloader would create
505 * one for us).
506 */
efi_allocate_bootparams(efi_handle_t handle,struct boot_params ** bp)507 static efi_status_t efi_allocate_bootparams(efi_handle_t handle,
508 struct boot_params **bp)
509 {
510 efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
511 struct boot_params *boot_params;
512 struct setup_header *hdr;
513 efi_status_t status;
514 unsigned long alloc;
515 char *cmdline_ptr;
516
517 status = efi_bs_call(handle_protocol, handle, &proto, (void **)&image);
518 if (status != EFI_SUCCESS) {
519 efi_err("Failed to get handle for LOADED_IMAGE_PROTOCOL\n");
520 return status;
521 }
522
523 status = efi_allocate_pages(PARAM_SIZE, &alloc, ULONG_MAX);
524 if (status != EFI_SUCCESS)
525 return status;
526
527 boot_params = memset((void *)alloc, 0x0, PARAM_SIZE);
528 hdr = &boot_params->hdr;
529
530 /* Assign the setup_header fields that the kernel actually cares about */
531 hdr->root_flags = 1;
532 hdr->vid_mode = 0xffff;
533
534 hdr->type_of_loader = 0x21;
535 hdr->initrd_addr_max = INT_MAX;
536
537 /* Convert unicode cmdline to ascii */
538 cmdline_ptr = efi_convert_cmdline(image);
539 if (!cmdline_ptr) {
540 efi_free(PARAM_SIZE, alloc);
541 return EFI_OUT_OF_RESOURCES;
542 }
543
544 efi_set_u64_split((unsigned long)cmdline_ptr, &hdr->cmd_line_ptr,
545 &boot_params->ext_cmd_line_ptr);
546
547 *bp = boot_params;
548 return EFI_SUCCESS;
549 }
550
add_e820ext(struct boot_params * params,struct setup_data * e820ext,u32 nr_entries)551 static void add_e820ext(struct boot_params *params,
552 struct setup_data *e820ext, u32 nr_entries)
553 {
554 struct setup_data *data;
555
556 e820ext->type = SETUP_E820_EXT;
557 e820ext->len = nr_entries * sizeof(struct boot_e820_entry);
558 e820ext->next = 0;
559
560 data = (struct setup_data *)(unsigned long)params->hdr.setup_data;
561
562 while (data && data->next)
563 data = (struct setup_data *)(unsigned long)data->next;
564
565 if (data)
566 data->next = (unsigned long)e820ext;
567 else
568 params->hdr.setup_data = (unsigned long)e820ext;
569 }
570
571 static efi_status_t
setup_e820(struct boot_params * params,struct setup_data * e820ext,u32 e820ext_size)572 setup_e820(struct boot_params *params, struct setup_data *e820ext, u32 e820ext_size)
573 {
574 struct boot_e820_entry *entry = params->e820_table;
575 struct efi_info *efi = ¶ms->efi_info;
576 struct boot_e820_entry *prev = NULL;
577 u32 nr_entries;
578 u32 nr_desc;
579 int i;
580
581 nr_entries = 0;
582 nr_desc = efi->efi_memmap_size / efi->efi_memdesc_size;
583
584 for (i = 0; i < nr_desc; i++) {
585 efi_memory_desc_t *d;
586 unsigned int e820_type = 0;
587 unsigned long m = efi->efi_memmap;
588
589 #ifdef CONFIG_X86_64
590 m |= (u64)efi->efi_memmap_hi << 32;
591 #endif
592
593 d = efi_memdesc_ptr(m, efi->efi_memdesc_size, i);
594 switch (d->type) {
595 case EFI_RESERVED_TYPE:
596 case EFI_RUNTIME_SERVICES_CODE:
597 case EFI_RUNTIME_SERVICES_DATA:
598 case EFI_MEMORY_MAPPED_IO:
599 case EFI_MEMORY_MAPPED_IO_PORT_SPACE:
600 case EFI_PAL_CODE:
601 e820_type = E820_TYPE_RESERVED;
602 break;
603
604 case EFI_UNUSABLE_MEMORY:
605 e820_type = E820_TYPE_UNUSABLE;
606 break;
607
608 case EFI_ACPI_RECLAIM_MEMORY:
609 e820_type = E820_TYPE_ACPI;
610 break;
611
612 case EFI_LOADER_CODE:
613 case EFI_LOADER_DATA:
614 case EFI_BOOT_SERVICES_CODE:
615 case EFI_BOOT_SERVICES_DATA:
616 case EFI_CONVENTIONAL_MEMORY:
617 if (efi_soft_reserve_enabled() &&
618 (d->attribute & EFI_MEMORY_SP))
619 e820_type = E820_TYPE_SOFT_RESERVED;
620 else
621 e820_type = E820_TYPE_RAM;
622 break;
623
624 case EFI_ACPI_MEMORY_NVS:
625 e820_type = E820_TYPE_NVS;
626 break;
627
628 case EFI_PERSISTENT_MEMORY:
629 e820_type = E820_TYPE_PMEM;
630 break;
631
632 case EFI_UNACCEPTED_MEMORY:
633 if (!IS_ENABLED(CONFIG_UNACCEPTED_MEMORY))
634 continue;
635 e820_type = E820_TYPE_RAM;
636 process_unaccepted_memory(d->phys_addr,
637 d->phys_addr + PAGE_SIZE * d->num_pages);
638 break;
639 default:
640 continue;
641 }
642
643 /* Merge adjacent mappings */
644 if (prev && prev->type == e820_type &&
645 (prev->addr + prev->size) == d->phys_addr) {
646 prev->size += d->num_pages << 12;
647 continue;
648 }
649
650 if (nr_entries == ARRAY_SIZE(params->e820_table)) {
651 u32 need = (nr_desc - i) * sizeof(struct e820_entry) +
652 sizeof(struct setup_data);
653
654 if (!e820ext || e820ext_size < need)
655 return EFI_BUFFER_TOO_SMALL;
656
657 /* boot_params map full, switch to e820 extended */
658 entry = (struct boot_e820_entry *)e820ext->data;
659 }
660
661 entry->addr = d->phys_addr;
662 entry->size = d->num_pages << PAGE_SHIFT;
663 entry->type = e820_type;
664 prev = entry++;
665 nr_entries++;
666 }
667
668 if (nr_entries > ARRAY_SIZE(params->e820_table)) {
669 u32 nr_e820ext = nr_entries - ARRAY_SIZE(params->e820_table);
670
671 add_e820ext(params, e820ext, nr_e820ext);
672 nr_entries -= nr_e820ext;
673 }
674
675 params->e820_entries = (u8)nr_entries;
676
677 return EFI_SUCCESS;
678 }
679
alloc_e820ext(u32 nr_desc,struct setup_data ** e820ext,u32 * e820ext_size)680 static efi_status_t alloc_e820ext(u32 nr_desc, struct setup_data **e820ext,
681 u32 *e820ext_size)
682 {
683 efi_status_t status;
684 unsigned long size;
685
686 size = sizeof(struct setup_data) +
687 sizeof(struct e820_entry) * nr_desc;
688
689 if (*e820ext) {
690 efi_bs_call(free_pool, *e820ext);
691 *e820ext = NULL;
692 *e820ext_size = 0;
693 }
694
695 status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size,
696 (void **)e820ext);
697 if (status == EFI_SUCCESS)
698 *e820ext_size = size;
699
700 return status;
701 }
702
allocate_e820(struct boot_params * params,struct setup_data ** e820ext,u32 * e820ext_size)703 static efi_status_t allocate_e820(struct boot_params *params,
704 struct setup_data **e820ext,
705 u32 *e820ext_size)
706 {
707 struct efi_boot_memmap *map __free(efi_pool) = NULL;
708 efi_status_t status;
709 __u32 nr_desc;
710
711 status = efi_get_memory_map(&map, false);
712 if (status != EFI_SUCCESS)
713 return status;
714
715 nr_desc = map->map_size / map->desc_size;
716 if (nr_desc > ARRAY_SIZE(params->e820_table) - EFI_MMAP_NR_SLACK_SLOTS) {
717 u32 nr_e820ext = nr_desc - ARRAY_SIZE(params->e820_table) +
718 EFI_MMAP_NR_SLACK_SLOTS;
719
720 status = alloc_e820ext(nr_e820ext, e820ext, e820ext_size);
721 if (status != EFI_SUCCESS)
722 return status;
723 }
724
725 if (IS_ENABLED(CONFIG_UNACCEPTED_MEMORY))
726 return allocate_unaccepted_bitmap(nr_desc, map);
727
728 return EFI_SUCCESS;
729 }
730
731 struct exit_boot_struct {
732 struct boot_params *boot_params;
733 struct efi_info *efi;
734 };
735
exit_boot_func(struct efi_boot_memmap * map,void * priv)736 static efi_status_t exit_boot_func(struct efi_boot_memmap *map,
737 void *priv)
738 {
739 const char *signature;
740 struct exit_boot_struct *p = priv;
741
742 signature = efi_is_64bit() ? EFI64_LOADER_SIGNATURE
743 : EFI32_LOADER_SIGNATURE;
744 memcpy(&p->efi->efi_loader_signature, signature, sizeof(__u32));
745
746 efi_set_u64_split((unsigned long)efi_system_table,
747 &p->efi->efi_systab, &p->efi->efi_systab_hi);
748 p->efi->efi_memdesc_size = map->desc_size;
749 p->efi->efi_memdesc_version = map->desc_ver;
750 efi_set_u64_split((unsigned long)map->map,
751 &p->efi->efi_memmap, &p->efi->efi_memmap_hi);
752 p->efi->efi_memmap_size = map->map_size;
753
754 return EFI_SUCCESS;
755 }
756
exit_boot(struct boot_params * boot_params,void * handle)757 static efi_status_t exit_boot(struct boot_params *boot_params, void *handle)
758 {
759 struct setup_data *e820ext = NULL;
760 __u32 e820ext_size = 0;
761 efi_status_t status;
762 struct exit_boot_struct priv;
763
764 priv.boot_params = boot_params;
765 priv.efi = &boot_params->efi_info;
766
767 status = allocate_e820(boot_params, &e820ext, &e820ext_size);
768 if (status != EFI_SUCCESS)
769 return status;
770
771 /* Might as well exit boot services now */
772 status = efi_exit_boot_services(handle, &priv, exit_boot_func);
773 if (status != EFI_SUCCESS)
774 return status;
775
776 /* Historic? */
777 boot_params->alt_mem_k = 32 * 1024;
778
779 status = setup_e820(boot_params, e820ext, e820ext_size);
780 if (status != EFI_SUCCESS)
781 return status;
782
783 return EFI_SUCCESS;
784 }
785
have_unsupported_snp_features(void)786 static bool have_unsupported_snp_features(void)
787 {
788 u64 unsupported;
789
790 unsupported = snp_get_unsupported_features(sev_get_status());
791 if (unsupported) {
792 efi_err("Unsupported SEV-SNP features detected: 0x%llx\n",
793 unsupported);
794 return true;
795 }
796 return false;
797 }
798
efi_get_seed(void * seed,int size)799 static void efi_get_seed(void *seed, int size)
800 {
801 efi_get_random_bytes(size, seed);
802
803 /*
804 * This only updates seed[0] when running on 32-bit, but in that case,
805 * seed[1] is not used anyway, as there is no virtual KASLR on 32-bit.
806 */
807 *(unsigned long *)seed ^= kaslr_get_random_long("EFI");
808 }
809
error(char * str)810 static void error(char *str)
811 {
812 efi_warn("Decompression failed: %s\n", str);
813 }
814
815 static const char *cmdline_memmap_override;
816
parse_options(const char * cmdline)817 static efi_status_t parse_options(const char *cmdline)
818 {
819 static const char opts[][14] = {
820 "mem=", "memmap=", "hugepages="
821 };
822
823 for (int i = 0; i < ARRAY_SIZE(opts); i++) {
824 const char *p = strstr(cmdline, opts[i]);
825
826 if (p == cmdline || (p > cmdline && isspace(p[-1]))) {
827 cmdline_memmap_override = opts[i];
828 break;
829 }
830 }
831
832 return efi_parse_options(cmdline);
833 }
834
efi_decompress_kernel(unsigned long * kernel_entry,struct boot_params * boot_params)835 static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry,
836 struct boot_params *boot_params)
837 {
838 unsigned long virt_addr = LOAD_PHYSICAL_ADDR;
839 unsigned long addr, alloc_size, entry;
840 efi_status_t status;
841 u32 seed[2] = {};
842
843 boot_params_ptr = boot_params;
844
845 /* determine the required size of the allocation */
846 alloc_size = ALIGN(max_t(unsigned long, output_len, kernel_total_size),
847 MIN_KERNEL_ALIGN);
848
849 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && !efi_nokaslr) {
850 u64 range = KERNEL_IMAGE_SIZE - LOAD_PHYSICAL_ADDR - kernel_total_size;
851 static const efi_char16_t ami[] = L"American Megatrends";
852
853 efi_get_seed(seed, sizeof(seed));
854
855 virt_addr += (range * seed[1]) >> 32;
856 virt_addr &= ~(CONFIG_PHYSICAL_ALIGN - 1);
857
858 /*
859 * Older Dell systems with AMI UEFI firmware v2.0 may hang
860 * while decompressing the kernel if physical address
861 * randomization is enabled.
862 *
863 * https://bugzilla.kernel.org/show_bug.cgi?id=218173
864 */
865 if (efi_system_table->hdr.revision <= EFI_2_00_SYSTEM_TABLE_REVISION &&
866 !memcmp(efistub_fw_vendor(), ami, sizeof(ami))) {
867 efi_debug("AMI firmware v2.0 or older detected - disabling physical KASLR\n");
868 seed[0] = 0;
869 } else if (cmdline_memmap_override) {
870 efi_info("%s detected on the kernel command line - disabling physical KASLR\n",
871 cmdline_memmap_override);
872 seed[0] = 0;
873 }
874
875 boot_params->hdr.loadflags |= KASLR_FLAG;
876 }
877
878 status = efi_random_alloc(alloc_size, CONFIG_PHYSICAL_ALIGN, &addr,
879 seed[0], EFI_LOADER_CODE,
880 LOAD_PHYSICAL_ADDR,
881 EFI_X86_KERNEL_ALLOC_LIMIT);
882 if (status != EFI_SUCCESS)
883 return status;
884
885 entry = decompress_kernel((void *)addr, virt_addr, error);
886 if (entry == ULONG_MAX) {
887 efi_free(alloc_size, addr);
888 return EFI_LOAD_ERROR;
889 }
890
891 *kernel_entry = addr + entry;
892
893 return efi_adjust_memory_range_protection(addr, kernel_text_size) ?:
894 efi_adjust_memory_range_protection(addr + kernel_inittext_offset,
895 kernel_inittext_size);
896 }
897
enter_kernel(unsigned long kernel_addr,struct boot_params * boot_params)898 static void __noreturn enter_kernel(unsigned long kernel_addr,
899 struct boot_params *boot_params)
900 {
901 /* enter decompressed kernel with boot_params pointer in RSI/ESI */
902 asm("jmp *%0"::"r"(kernel_addr), "S"(boot_params));
903
904 unreachable();
905 }
906
907 /*
908 * On success, this routine will jump to the relocated image directly and never
909 * return. On failure, it will exit to the firmware via efi_exit() instead of
910 * returning.
911 */
efi_stub_entry(efi_handle_t handle,efi_system_table_t * sys_table_arg,struct boot_params * boot_params)912 void __noreturn efi_stub_entry(efi_handle_t handle,
913 efi_system_table_t *sys_table_arg,
914 struct boot_params *boot_params)
915
916 {
917 efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID;
918 const struct linux_efi_initrd *initrd = NULL;
919 unsigned long kernel_entry;
920 struct setup_header *hdr;
921 efi_status_t status;
922
923 efi_system_table = sys_table_arg;
924 /* Check if we were booted by the EFI firmware */
925 if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE)
926 efi_exit(handle, EFI_INVALID_PARAMETER);
927
928 if (!IS_ENABLED(CONFIG_EFI_HANDOVER_PROTOCOL) || !boot_params) {
929 status = efi_allocate_bootparams(handle, &boot_params);
930 if (status != EFI_SUCCESS)
931 efi_exit(handle, status);
932 }
933
934 hdr = &boot_params->hdr;
935
936 if (have_unsupported_snp_features())
937 efi_exit(handle, EFI_UNSUPPORTED);
938
939 if (IS_ENABLED(CONFIG_EFI_DXE_MEM_ATTRIBUTES)) {
940 efi_dxe_table = get_efi_config_table(EFI_DXE_SERVICES_TABLE_GUID);
941 if (efi_dxe_table &&
942 efi_dxe_table->hdr.signature != EFI_DXE_SERVICES_TABLE_SIGNATURE) {
943 efi_warn("Ignoring DXE services table: invalid signature\n");
944 efi_dxe_table = NULL;
945 }
946 }
947
948 /* grab the memory attributes protocol if it exists */
949 efi_bs_call(locate_protocol, &guid, NULL, (void **)&memattr);
950
951 status = efi_setup_5level_paging();
952 if (status != EFI_SUCCESS) {
953 efi_err("efi_setup_5level_paging() failed!\n");
954 goto fail;
955 }
956
957 #ifdef CONFIG_CMDLINE_BOOL
958 status = parse_options(CONFIG_CMDLINE);
959 if (status != EFI_SUCCESS) {
960 efi_err("Failed to parse options\n");
961 goto fail;
962 }
963 #endif
964 if (!IS_ENABLED(CONFIG_CMDLINE_OVERRIDE)) {
965 unsigned long cmdline_paddr = ((u64)hdr->cmd_line_ptr |
966 ((u64)boot_params->ext_cmd_line_ptr << 32));
967 status = parse_options((char *)cmdline_paddr);
968 if (status != EFI_SUCCESS) {
969 efi_err("Failed to parse options\n");
970 goto fail;
971 }
972 }
973
974 if (efi_mem_encrypt > 0)
975 hdr->xloadflags |= XLF_MEM_ENCRYPTION;
976
977 status = efi_decompress_kernel(&kernel_entry, boot_params);
978 if (status != EFI_SUCCESS) {
979 efi_err("Failed to decompress kernel\n");
980 goto fail;
981 }
982
983 /*
984 * At this point, an initrd may already have been loaded by the
985 * bootloader and passed via bootparams. We permit an initrd loaded
986 * from the LINUX_EFI_INITRD_MEDIA_GUID device path to supersede it.
987 *
988 * If the device path is not present, any command-line initrd=
989 * arguments will be processed only if image is not NULL, which will be
990 * the case only if we were loaded via the PE entry point.
991 */
992 status = efi_load_initrd(image, hdr->initrd_addr_max, ULONG_MAX,
993 &initrd);
994 if (status != EFI_SUCCESS)
995 goto fail;
996 if (initrd && initrd->size > 0) {
997 efi_set_u64_split(initrd->base, &hdr->ramdisk_image,
998 &boot_params->ext_ramdisk_image);
999 efi_set_u64_split(initrd->size, &hdr->ramdisk_size,
1000 &boot_params->ext_ramdisk_size);
1001 }
1002
1003
1004 /*
1005 * If the boot loader gave us a value for secure_boot then we use that,
1006 * otherwise we ask the BIOS.
1007 */
1008 if (boot_params->secure_boot == efi_secureboot_mode_unset)
1009 boot_params->secure_boot = efi_get_secureboot();
1010
1011 /* Ask the firmware to clear memory on unclean shutdown */
1012 efi_enable_reset_attack_mitigation();
1013
1014 efi_random_get_seed();
1015
1016 efi_retrieve_eventlog();
1017
1018 setup_graphics(boot_params);
1019
1020 setup_efi_pci(boot_params);
1021
1022 setup_quirks(boot_params);
1023
1024 setup_unaccepted_memory();
1025
1026 status = exit_boot(boot_params, handle);
1027 if (status != EFI_SUCCESS) {
1028 efi_err("exit_boot() failed!\n");
1029 goto fail;
1030 }
1031
1032 /*
1033 * Call the SEV init code while still running with the firmware's
1034 * GDT/IDT, so #VC exceptions will be handled by EFI.
1035 */
1036 sev_enable(boot_params);
1037
1038 efi_5level_switch();
1039
1040 enter_kernel(kernel_entry, boot_params);
1041 fail:
1042 efi_err("efi_stub_entry() failed!\n");
1043
1044 efi_exit(handle, status);
1045 }
1046
efi_pe_entry(efi_handle_t handle,efi_system_table_t * sys_table_arg)1047 efi_status_t __efiapi efi_pe_entry(efi_handle_t handle,
1048 efi_system_table_t *sys_table_arg)
1049 {
1050 efi_stub_entry(handle, sys_table_arg, NULL);
1051 }
1052
1053 #ifdef CONFIG_EFI_HANDOVER_PROTOCOL
efi_handover_entry(efi_handle_t handle,efi_system_table_t * sys_table_arg,struct boot_params * boot_params)1054 void efi_handover_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
1055 struct boot_params *boot_params)
1056 {
1057 memset(_bss, 0, _ebss - _bss);
1058 efi_stub_entry(handle, sys_table_arg, boot_params);
1059 }
1060
1061 #ifndef CONFIG_EFI_MIXED
1062 extern __alias(efi_handover_entry)
1063 void efi32_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
1064 struct boot_params *boot_params);
1065
1066 extern __alias(efi_handover_entry)
1067 void efi64_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
1068 struct boot_params *boot_params);
1069 #endif
1070 #endif
1071