1 // SPDX-License-Identifier: GPL-2.0
2 #define boot_fmt(fmt) "physmem: " fmt
3 #include <linux/processor.h>
4 #include <linux/errno.h>
5 #include <linux/init.h>
6 #include <asm/physmem_info.h>
7 #include <asm/stacktrace.h>
8 #include <asm/boot_data.h>
9 #include <asm/sparsemem.h>
10 #include <asm/sections.h>
11 #include <asm/setup.h>
12 #include <asm/sclp.h>
13 #include <asm/asm.h>
14 #include <asm/uv.h>
15 #include "decompressor.h"
16 #include "boot.h"
17
18 struct physmem_info __bootdata(physmem_info);
19 static unsigned int physmem_alloc_ranges;
20 static unsigned long physmem_alloc_pos;
21
22 /* up to 256 storage elements, 1020 subincrements each */
23 #define ENTRIES_EXTENDED_MAX \
24 (256 * (1020 / 2) * sizeof(struct physmem_range))
25
__get_physmem_range_ptr(u32 n)26 static struct physmem_range *__get_physmem_range_ptr(u32 n)
27 {
28 if (n < MEM_INLINED_ENTRIES)
29 return &physmem_info.online[n];
30 if (unlikely(!physmem_info.online_extended)) {
31 physmem_info.online_extended = (struct physmem_range *)physmem_alloc_range(
32 RR_MEM_DETECT_EXT, ENTRIES_EXTENDED_MAX, sizeof(long), 0,
33 physmem_alloc_pos, true);
34 }
35 return &physmem_info.online_extended[n - MEM_INLINED_ENTRIES];
36 }
37
38 /*
39 * sequential calls to add_physmem_online_range with adjacent memory ranges
40 * are merged together into single memory range.
41 */
add_physmem_online_range(u64 start,u64 end)42 void add_physmem_online_range(u64 start, u64 end)
43 {
44 struct physmem_range *range;
45
46 if (physmem_info.range_count) {
47 range = __get_physmem_range_ptr(physmem_info.range_count - 1);
48 if (range->end == start) {
49 range->end = end;
50 return;
51 }
52 }
53
54 range = __get_physmem_range_ptr(physmem_info.range_count);
55 range->start = start;
56 range->end = end;
57 physmem_info.range_count++;
58 }
59
__diag260(unsigned long rx1,unsigned long rx2)60 static int __diag260(unsigned long rx1, unsigned long rx2)
61 {
62 union register_pair rx;
63 int cc, exception;
64 unsigned long ry;
65
66 rx.even = rx1;
67 rx.odd = rx2;
68 ry = 0x10; /* storage configuration */
69 exception = 1;
70 asm_inline volatile(
71 " diag %[rx],%[ry],0x260\n"
72 "0: lhi %[exc],0\n"
73 "1:\n"
74 CC_IPM(cc)
75 EX_TABLE(0b, 1b)
76 : CC_OUT(cc, cc), [exc] "+d" (exception), [ry] "+d" (ry)
77 : [rx] "d" (rx.pair)
78 : CC_CLOBBER_LIST("memory"));
79 cc = exception ? -1 : CC_TRANSFORM(cc);
80 return cc == 0 ? ry : -1;
81 }
82
diag260(void)83 static int diag260(void)
84 {
85 int rc, i;
86
87 struct {
88 unsigned long start;
89 unsigned long end;
90 } storage_extents[8] __aligned(16); /* VM supports up to 8 extends */
91
92 memset(storage_extents, 0, sizeof(storage_extents));
93 rc = __diag260((unsigned long)storage_extents, sizeof(storage_extents));
94 if (rc == -1)
95 return -1;
96
97 for (i = 0; i < min_t(int, rc, ARRAY_SIZE(storage_extents)); i++)
98 add_physmem_online_range(storage_extents[i].start, storage_extents[i].end + 1);
99 return 0;
100 }
101
102 #define DIAG500_SC_STOR_LIMIT 4
103
diag500_storage_limit(unsigned long * max_physmem_end)104 static int diag500_storage_limit(unsigned long *max_physmem_end)
105 {
106 unsigned long storage_limit;
107
108 asm_inline volatile(
109 " lghi %%r1,%[subcode]\n"
110 " lghi %%r2,0\n"
111 " diag %%r2,%%r4,0x500\n"
112 "0: lgr %[slimit],%%r2\n"
113 EX_TABLE(0b, 0b)
114 : [slimit] "=d" (storage_limit)
115 : [subcode] "i" (DIAG500_SC_STOR_LIMIT)
116 : "memory", "1", "2");
117 if (!storage_limit)
118 return -EINVAL;
119 /* Convert inclusive end to exclusive end */
120 *max_physmem_end = storage_limit + 1;
121 return 0;
122 }
123
tprot(unsigned long addr)124 static int tprot(unsigned long addr)
125 {
126 int cc, exception;
127
128 exception = 1;
129 asm_inline volatile(
130 " tprot 0(%[addr]),0\n"
131 "0: lhi %[exc],0\n"
132 "1:\n"
133 CC_IPM(cc)
134 EX_TABLE(0b, 1b)
135 : CC_OUT(cc, cc), [exc] "+d" (exception)
136 : [addr] "a" (addr)
137 : CC_CLOBBER_LIST("memory"));
138 cc = exception ? -EFAULT : CC_TRANSFORM(cc);
139 return cc;
140 }
141
search_mem_end(void)142 static unsigned long search_mem_end(void)
143 {
144 unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */
145 unsigned long offset = 0;
146 unsigned long pivot;
147
148 while (range > 1) {
149 range >>= 1;
150 pivot = offset + range;
151 if (!tprot(pivot << 20))
152 offset = pivot;
153 }
154 return (offset + 1) << 20;
155 }
156
detect_max_physmem_end(void)157 unsigned long detect_max_physmem_end(void)
158 {
159 unsigned long max_physmem_end = 0;
160
161 if (!diag500_storage_limit(&max_physmem_end)) {
162 physmem_info.info_source = MEM_DETECT_DIAG500_STOR_LIMIT;
163 } else if (!sclp_early_get_memsize(&max_physmem_end)) {
164 physmem_info.info_source = MEM_DETECT_SCLP_READ_INFO;
165 } else {
166 max_physmem_end = search_mem_end();
167 physmem_info.info_source = MEM_DETECT_BIN_SEARCH;
168 }
169 boot_debug("Max physical memory: 0x%016lx (info source: %s)\n", max_physmem_end,
170 get_physmem_info_source());
171 return max_physmem_end;
172 }
173
detect_physmem_online_ranges(unsigned long max_physmem_end)174 void detect_physmem_online_ranges(unsigned long max_physmem_end)
175 {
176 unsigned long start, end;
177 int i;
178
179 if (!sclp_early_read_storage_info()) {
180 physmem_info.info_source = MEM_DETECT_SCLP_STOR_INFO;
181 } else if (physmem_info.info_source == MEM_DETECT_DIAG500_STOR_LIMIT) {
182 unsigned long online_end;
183
184 if (!sclp_early_get_memsize(&online_end)) {
185 physmem_info.info_source = MEM_DETECT_SCLP_READ_INFO;
186 add_physmem_online_range(0, online_end);
187 }
188 } else if (!diag260()) {
189 physmem_info.info_source = MEM_DETECT_DIAG260;
190 } else if (max_physmem_end) {
191 add_physmem_online_range(0, max_physmem_end);
192 }
193 boot_debug("Online memory ranges (info source: %s):\n", get_physmem_info_source());
194 for_each_physmem_online_range(i, &start, &end)
195 boot_debug(" online [%d]: 0x%016lx-0x%016lx\n", i, start, end);
196 }
197
physmem_set_usable_limit(unsigned long limit)198 void physmem_set_usable_limit(unsigned long limit)
199 {
200 physmem_info.usable = limit;
201 physmem_alloc_pos = limit;
202 boot_debug("Usable memory limit: 0x%016lx\n", limit);
203 }
204
die_oom(unsigned long size,unsigned long align,unsigned long min,unsigned long max)205 static void die_oom(unsigned long size, unsigned long align, unsigned long min, unsigned long max)
206 {
207 unsigned long start, end, total_mem = 0, total_reserved_mem = 0;
208 struct reserved_range *range;
209 enum reserved_range_type t;
210 int i;
211
212 boot_emerg("Linux version %s\n", kernel_version);
213 if (!is_prot_virt_guest() && early_command_line[0])
214 boot_emerg("Kernel command line: %s\n", early_command_line);
215 boot_emerg("Out of memory allocating %lu bytes 0x%lx aligned in range %lx:%lx\n",
216 size, align, min, max);
217 boot_emerg("Reserved memory ranges:\n");
218 for_each_physmem_reserved_range(t, range, &start, &end) {
219 boot_emerg("%016lx %016lx %s\n", start, end, get_rr_type_name(t));
220 total_reserved_mem += end - start;
221 }
222 boot_emerg("Usable online memory ranges (info source: %s [%d]):\n",
223 get_physmem_info_source(), physmem_info.info_source);
224 for_each_physmem_usable_range(i, &start, &end) {
225 boot_emerg("%016lx %016lx\n", start, end);
226 total_mem += end - start;
227 }
228 boot_emerg("Usable online memory total: %lu Reserved: %lu Free: %lu\n",
229 total_mem, total_reserved_mem,
230 total_mem > total_reserved_mem ? total_mem - total_reserved_mem : 0);
231 print_stacktrace(current_frame_address());
232 boot_emerg(" -- System halted\n");
233 disabled_wait();
234 }
235
_physmem_reserve(enum reserved_range_type type,unsigned long addr,unsigned long size)236 static void _physmem_reserve(enum reserved_range_type type, unsigned long addr, unsigned long size)
237 {
238 physmem_info.reserved[type].start = addr;
239 physmem_info.reserved[type].end = addr + size;
240 }
241
physmem_reserve(enum reserved_range_type type,unsigned long addr,unsigned long size)242 void physmem_reserve(enum reserved_range_type type, unsigned long addr, unsigned long size)
243 {
244 _physmem_reserve(type, addr, size);
245 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Reserve:", addr, addr + size,
246 get_rr_type_name(type));
247 }
248
physmem_free(enum reserved_range_type type)249 void physmem_free(enum reserved_range_type type)
250 {
251 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Free:", physmem_info.reserved[type].start,
252 physmem_info.reserved[type].end, get_rr_type_name(type));
253 physmem_info.reserved[type].start = 0;
254 physmem_info.reserved[type].end = 0;
255 }
256
__physmem_alloc_intersects(unsigned long addr,unsigned long size,unsigned long * intersection_start)257 static bool __physmem_alloc_intersects(unsigned long addr, unsigned long size,
258 unsigned long *intersection_start)
259 {
260 unsigned long res_addr, res_size;
261 int t;
262
263 for (t = 0; t < RR_MAX; t++) {
264 if (!get_physmem_reserved(t, &res_addr, &res_size))
265 continue;
266 if (intersects(addr, size, res_addr, res_size)) {
267 *intersection_start = res_addr;
268 return true;
269 }
270 }
271 return ipl_report_certs_intersects(addr, size, intersection_start);
272 }
273
__physmem_alloc_range(unsigned long size,unsigned long align,unsigned long min,unsigned long max,unsigned int from_ranges,unsigned int * ranges_left,bool die_on_oom)274 static unsigned long __physmem_alloc_range(unsigned long size, unsigned long align,
275 unsigned long min, unsigned long max,
276 unsigned int from_ranges, unsigned int *ranges_left,
277 bool die_on_oom)
278 {
279 unsigned int nranges = from_ranges ?: physmem_info.range_count;
280 unsigned long range_start, range_end;
281 unsigned long intersection_start;
282 unsigned long addr, pos = max;
283
284 align = max(align, 8UL);
285 while (nranges) {
286 __get_physmem_range(nranges - 1, &range_start, &range_end, false);
287 pos = min(range_end, pos);
288
289 if (round_up(min, align) + size > pos)
290 break;
291 addr = round_down(pos - size, align);
292 if (range_start > addr) {
293 nranges--;
294 continue;
295 }
296 if (__physmem_alloc_intersects(addr, size, &intersection_start)) {
297 pos = intersection_start;
298 continue;
299 }
300
301 if (ranges_left)
302 *ranges_left = nranges;
303 return addr;
304 }
305 if (die_on_oom)
306 die_oom(size, align, min, max);
307 return 0;
308 }
309
physmem_alloc_range(enum reserved_range_type type,unsigned long size,unsigned long align,unsigned long min,unsigned long max,bool die_on_oom)310 unsigned long physmem_alloc_range(enum reserved_range_type type, unsigned long size,
311 unsigned long align, unsigned long min, unsigned long max,
312 bool die_on_oom)
313 {
314 unsigned long addr;
315
316 max = min(max, physmem_alloc_pos);
317 addr = __physmem_alloc_range(size, align, min, max, 0, NULL, die_on_oom);
318 if (addr)
319 _physmem_reserve(type, addr, size);
320 boot_debug("%-14s 0x%016lx-0x%016lx %s\n", "Alloc range:", addr, addr + size,
321 get_rr_type_name(type));
322 return addr;
323 }
324
physmem_alloc(enum reserved_range_type type,unsigned long size,unsigned long align,bool die_on_oom)325 unsigned long physmem_alloc(enum reserved_range_type type, unsigned long size,
326 unsigned long align, bool die_on_oom)
327 {
328 struct reserved_range *range = &physmem_info.reserved[type];
329 struct reserved_range *new_range = NULL;
330 unsigned int ranges_left;
331 unsigned long addr;
332
333 addr = __physmem_alloc_range(size, align, 0, physmem_alloc_pos, physmem_alloc_ranges,
334 &ranges_left, die_on_oom);
335 if (!addr)
336 return 0;
337 /* if not a consecutive allocation of the same type or first allocation */
338 if (range->start != addr + size) {
339 if (range->end) {
340 addr = __physmem_alloc_range(sizeof(struct reserved_range), 0, 0,
341 physmem_alloc_pos, physmem_alloc_ranges,
342 &ranges_left, true);
343 new_range = (struct reserved_range *)addr;
344 addr = __physmem_alloc_range(size, align, 0, addr, ranges_left,
345 &ranges_left, die_on_oom);
346 if (!addr)
347 return 0;
348 *new_range = *range;
349 range->chain = new_range;
350 }
351 range->end = addr + size;
352 }
353 if (type != RR_VMEM) {
354 boot_debug("%-14s 0x%016lx-0x%016lx %-20s align 0x%lx split %d\n", "Alloc topdown:",
355 addr, addr + size, get_rr_type_name(type), align, !!new_range);
356 }
357 range->start = addr;
358 physmem_alloc_pos = addr;
359 physmem_alloc_ranges = ranges_left;
360 return addr;
361 }
362
physmem_alloc_or_die(enum reserved_range_type type,unsigned long size,unsigned long align)363 unsigned long physmem_alloc_or_die(enum reserved_range_type type, unsigned long size,
364 unsigned long align)
365 {
366 return physmem_alloc(type, size, align, true);
367 }
368
get_physmem_alloc_pos(void)369 unsigned long get_physmem_alloc_pos(void)
370 {
371 return physmem_alloc_pos;
372 }
373
dump_physmem_reserved(void)374 void dump_physmem_reserved(void)
375 {
376 struct reserved_range *range;
377 enum reserved_range_type t;
378 unsigned long start, end;
379
380 boot_debug("Reserved memory ranges:\n");
381 for_each_physmem_reserved_range(t, range, &start, &end) {
382 if (end) {
383 boot_debug("%-14s 0x%016lx-0x%016lx @%012lx chain %012lx\n",
384 get_rr_type_name(t), start, end, (unsigned long)range,
385 (unsigned long)range->chain);
386 }
387 }
388 }
389