1 // SPDX-License-Identifier: GPL-2.0 AND MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include <kunit/test.h> 7 #include <kunit/visibility.h> 8 9 #include "tests/xe_bo_test.h" 10 #include "tests/xe_pci_test.h" 11 #include "tests/xe_test.h" 12 13 #include "xe_bo_evict.h" 14 #include "xe_pci.h" 15 #include "xe_pm.h" 16 17 static int ccs_test_migrate(struct xe_gt *gt, struct xe_bo *bo, 18 bool clear, u64 get_val, u64 assign_val, 19 struct kunit *test) 20 { 21 struct dma_fence *fence; 22 struct ttm_tt *ttm; 23 struct page *page; 24 pgoff_t ccs_page; 25 long timeout; 26 u64 *cpu_map; 27 int ret; 28 u32 offset; 29 30 /* Move bo to VRAM if not already there. */ 31 ret = xe_bo_validate(bo, NULL, false); 32 if (ret) { 33 KUNIT_FAIL(test, "Failed to validate bo.\n"); 34 return ret; 35 } 36 37 /* Optionally clear bo *and* CCS data in VRAM. */ 38 if (clear) { 39 fence = xe_migrate_clear(gt_to_tile(gt)->migrate, bo, bo->ttm.resource); 40 if (IS_ERR(fence)) { 41 KUNIT_FAIL(test, "Failed to submit bo clear.\n"); 42 return PTR_ERR(fence); 43 } 44 dma_fence_put(fence); 45 } 46 47 /* Evict to system. CCS data should be copied. */ 48 ret = xe_bo_evict(bo, true); 49 if (ret) { 50 KUNIT_FAIL(test, "Failed to evict bo.\n"); 51 return ret; 52 } 53 54 /* Sync all migration blits */ 55 timeout = dma_resv_wait_timeout(bo->ttm.base.resv, 56 DMA_RESV_USAGE_KERNEL, 57 true, 58 5 * HZ); 59 if (timeout <= 0) { 60 KUNIT_FAIL(test, "Failed to sync bo eviction.\n"); 61 return -ETIME; 62 } 63 64 /* 65 * Bo with CCS data is now in system memory. Verify backing store 66 * and data integrity. Then assign for the next testing round while 67 * we still have a CPU map. 68 */ 69 ttm = bo->ttm.ttm; 70 if (!ttm || !ttm_tt_is_populated(ttm)) { 71 KUNIT_FAIL(test, "Bo was not in expected placement.\n"); 72 return -EINVAL; 73 } 74 75 ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT; 76 if (ccs_page >= ttm->num_pages) { 77 KUNIT_FAIL(test, "No TTM CCS pages present.\n"); 78 return -EINVAL; 79 } 80 81 page = ttm->pages[ccs_page]; 82 cpu_map = kmap_local_page(page); 83 84 /* Check first CCS value */ 85 if (cpu_map[0] != get_val) { 86 KUNIT_FAIL(test, 87 "Expected CCS readout 0x%016llx, got 0x%016llx.\n", 88 (unsigned long long)get_val, 89 (unsigned long long)cpu_map[0]); 90 ret = -EINVAL; 91 } 92 93 /* Check last CCS value, or at least last value in page. */ 94 offset = xe_device_ccs_bytes(gt_to_xe(gt), bo->size); 95 offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1; 96 if (cpu_map[offset] != get_val) { 97 KUNIT_FAIL(test, 98 "Expected CCS readout 0x%016llx, got 0x%016llx.\n", 99 (unsigned long long)get_val, 100 (unsigned long long)cpu_map[offset]); 101 ret = -EINVAL; 102 } 103 104 cpu_map[0] = assign_val; 105 cpu_map[offset] = assign_val; 106 kunmap_local(cpu_map); 107 108 return ret; 109 } 110 111 static void ccs_test_run_gt(struct xe_device *xe, struct xe_gt *gt, 112 struct kunit *test) 113 { 114 struct xe_bo *bo; 115 u32 vram_bit; 116 int ret; 117 118 /* TODO: Sanity check */ 119 vram_bit = XE_BO_CREATE_VRAM0_BIT << gt_to_tile(gt)->id; 120 kunit_info(test, "Testing gt id %u vram id %u\n", gt->info.id, 121 gt_to_tile(gt)->id); 122 123 bo = xe_bo_create_locked(xe, NULL, NULL, SZ_1M, ttm_bo_type_device, 124 vram_bit); 125 if (IS_ERR(bo)) { 126 KUNIT_FAIL(test, "Failed to create bo.\n"); 127 return; 128 } 129 130 kunit_info(test, "Verifying that CCS data is cleared on creation.\n"); 131 ret = ccs_test_migrate(gt, bo, false, 0ULL, 0xdeadbeefdeadbeefULL, 132 test); 133 if (ret) 134 goto out_unlock; 135 136 kunit_info(test, "Verifying that CCS data survives migration.\n"); 137 ret = ccs_test_migrate(gt, bo, false, 0xdeadbeefdeadbeefULL, 138 0xdeadbeefdeadbeefULL, test); 139 if (ret) 140 goto out_unlock; 141 142 kunit_info(test, "Verifying that CCS data can be properly cleared.\n"); 143 ret = ccs_test_migrate(gt, bo, true, 0ULL, 0ULL, test); 144 145 out_unlock: 146 xe_bo_unlock_no_vm(bo); 147 xe_bo_put(bo); 148 } 149 150 static int ccs_test_run_device(struct xe_device *xe) 151 { 152 struct kunit *test = xe_cur_kunit(); 153 struct xe_gt *gt; 154 int id; 155 156 if (!xe_device_has_flat_ccs(xe)) { 157 kunit_info(test, "Skipping non-flat-ccs device.\n"); 158 return 0; 159 } 160 161 xe_device_mem_access_get(xe); 162 163 for_each_gt(gt, xe, id) 164 ccs_test_run_gt(xe, gt, test); 165 166 xe_device_mem_access_put(xe); 167 168 return 0; 169 } 170 171 void xe_ccs_migrate_kunit(struct kunit *test) 172 { 173 xe_call_for_each_device(ccs_test_run_device); 174 } 175 EXPORT_SYMBOL_IF_KUNIT(xe_ccs_migrate_kunit); 176 177 static int evict_test_run_gt(struct xe_device *xe, struct xe_gt *gt, struct kunit *test) 178 { 179 struct xe_bo *bo, *external; 180 unsigned int bo_flags = XE_BO_CREATE_USER_BIT | 181 XE_BO_CREATE_VRAM_IF_DGFX(gt_to_tile(gt)); 182 struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate); 183 struct ww_acquire_ctx ww; 184 int err, i; 185 186 kunit_info(test, "Testing device %s gt id %u vram id %u\n", 187 dev_name(xe->drm.dev), gt->info.id, gt_to_tile(gt)->id); 188 189 for (i = 0; i < 2; ++i) { 190 xe_vm_lock(vm, &ww, 0, false); 191 bo = xe_bo_create(xe, NULL, vm, 0x10000, ttm_bo_type_device, 192 bo_flags); 193 xe_vm_unlock(vm, &ww); 194 if (IS_ERR(bo)) { 195 KUNIT_FAIL(test, "bo create err=%pe\n", bo); 196 break; 197 } 198 199 external = xe_bo_create(xe, NULL, NULL, 0x10000, 200 ttm_bo_type_device, bo_flags); 201 if (IS_ERR(external)) { 202 KUNIT_FAIL(test, "external bo create err=%pe\n", external); 203 goto cleanup_bo; 204 } 205 206 xe_bo_lock(external, &ww, 0, false); 207 err = xe_bo_pin_external(external); 208 xe_bo_unlock(external, &ww); 209 if (err) { 210 KUNIT_FAIL(test, "external bo pin err=%pe\n", 211 ERR_PTR(err)); 212 goto cleanup_external; 213 } 214 215 err = xe_bo_evict_all(xe); 216 if (err) { 217 KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err)); 218 goto cleanup_all; 219 } 220 221 err = xe_bo_restore_kernel(xe); 222 if (err) { 223 KUNIT_FAIL(test, "restore kernel err=%pe\n", 224 ERR_PTR(err)); 225 goto cleanup_all; 226 } 227 228 err = xe_bo_restore_user(xe); 229 if (err) { 230 KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err)); 231 goto cleanup_all; 232 } 233 234 if (!xe_bo_is_vram(external)) { 235 KUNIT_FAIL(test, "external bo is not vram\n"); 236 err = -EPROTO; 237 goto cleanup_all; 238 } 239 240 if (xe_bo_is_vram(bo)) { 241 KUNIT_FAIL(test, "bo is vram\n"); 242 err = -EPROTO; 243 goto cleanup_all; 244 } 245 246 if (i) { 247 down_read(&vm->lock); 248 xe_vm_lock(vm, &ww, 0, false); 249 err = xe_bo_validate(bo, bo->vm, false); 250 xe_vm_unlock(vm, &ww); 251 up_read(&vm->lock); 252 if (err) { 253 KUNIT_FAIL(test, "bo valid err=%pe\n", 254 ERR_PTR(err)); 255 goto cleanup_all; 256 } 257 xe_bo_lock(external, &ww, 0, false); 258 err = xe_bo_validate(external, NULL, false); 259 xe_bo_unlock(external, &ww); 260 if (err) { 261 KUNIT_FAIL(test, "external bo valid err=%pe\n", 262 ERR_PTR(err)); 263 goto cleanup_all; 264 } 265 } 266 267 xe_bo_lock(external, &ww, 0, false); 268 xe_bo_unpin_external(external); 269 xe_bo_unlock(external, &ww); 270 271 xe_bo_put(external); 272 xe_bo_put(bo); 273 continue; 274 275 cleanup_all: 276 xe_bo_lock(external, &ww, 0, false); 277 xe_bo_unpin_external(external); 278 xe_bo_unlock(external, &ww); 279 cleanup_external: 280 xe_bo_put(external); 281 cleanup_bo: 282 xe_bo_put(bo); 283 break; 284 } 285 286 xe_vm_put(vm); 287 288 return 0; 289 } 290 291 static int evict_test_run_device(struct xe_device *xe) 292 { 293 struct kunit *test = xe_cur_kunit(); 294 struct xe_gt *gt; 295 int id; 296 297 if (!IS_DGFX(xe)) { 298 kunit_info(test, "Skipping non-discrete device %s.\n", 299 dev_name(xe->drm.dev)); 300 return 0; 301 } 302 303 xe_device_mem_access_get(xe); 304 305 for_each_gt(gt, xe, id) 306 evict_test_run_gt(xe, gt, test); 307 308 xe_device_mem_access_put(xe); 309 310 return 0; 311 } 312 313 void xe_bo_evict_kunit(struct kunit *test) 314 { 315 xe_call_for_each_device(evict_test_run_device); 316 } 317 EXPORT_SYMBOL_IF_KUNIT(xe_bo_evict_kunit); 318