1# Test utilities for fetching & caching assets 2# 3# Copyright 2024 Red Hat, Inc. 4# 5# This work is licensed under the terms of the GNU GPL, version 2 or 6# later. See the COPYING file in the top-level directory. 7 8import hashlib 9import logging 10import os 11import stat 12import sys 13import unittest 14import urllib.request 15from time import sleep 16from pathlib import Path 17from shutil import copyfileobj 18from urllib.error import HTTPError 19 20 21# Instances of this class must be declared as class level variables 22# starting with a name "ASSET_". This enables the pre-caching logic 23# to easily find all referenced assets and download them prior to 24# execution of the tests. 25class Asset: 26 27 def __init__(self, url, hashsum): 28 self.url = url 29 self.hash = hashsum 30 cache_dir_env = os.getenv('QEMU_TEST_CACHE_DIR') 31 if cache_dir_env: 32 self.cache_dir = Path(cache_dir_env, "download") 33 else: 34 self.cache_dir = Path(Path("~").expanduser(), 35 ".cache", "qemu", "download") 36 self.cache_file = Path(self.cache_dir, hashsum) 37 self.log = logging.getLogger('qemu-test') 38 39 def __repr__(self): 40 return "Asset: url=%s hash=%s cache=%s" % ( 41 self.url, self.hash, self.cache_file) 42 43 def __str__(self): 44 return str(self.cache_file) 45 46 def _check(self, cache_file): 47 if self.hash is None: 48 return True 49 if len(self.hash) == 64: 50 hl = hashlib.sha256() 51 elif len(self.hash) == 128: 52 hl = hashlib.sha512() 53 else: 54 raise Exception("unknown hash type") 55 56 # Calculate the hash of the file: 57 with open(cache_file, 'rb') as file: 58 while True: 59 chunk = file.read(1 << 20) 60 if not chunk: 61 break 62 hl.update(chunk) 63 64 return self.hash == hl.hexdigest() 65 66 def valid(self): 67 return self.cache_file.exists() and self._check(self.cache_file) 68 69 def fetchable(self): 70 return not os.environ.get("QEMU_TEST_NO_DOWNLOAD", False) 71 72 def available(self): 73 return self.valid() or self.fetchable() 74 75 def _wait_for_other_download(self, tmp_cache_file): 76 # Another thread already seems to download the asset, so wait until 77 # it is done, while also checking the size to see whether it is stuck 78 try: 79 current_size = tmp_cache_file.stat().st_size 80 new_size = current_size 81 except: 82 if os.path.exists(self.cache_file): 83 return True 84 raise 85 waittime = lastchange = 600 86 while waittime > 0: 87 sleep(1) 88 waittime -= 1 89 try: 90 new_size = tmp_cache_file.stat().st_size 91 except: 92 if os.path.exists(self.cache_file): 93 return True 94 raise 95 if new_size != current_size: 96 lastchange = waittime 97 current_size = new_size 98 elif lastchange - waittime > 90: 99 return False 100 101 self.log.debug("Time out while waiting for %s!", tmp_cache_file) 102 raise 103 104 def fetch(self): 105 if not self.cache_dir.exists(): 106 self.cache_dir.mkdir(parents=True, exist_ok=True) 107 108 if self.valid(): 109 self.log.debug("Using cached asset %s for %s", 110 self.cache_file, self.url) 111 return str(self.cache_file) 112 113 if not self.fetchable(): 114 raise Exception("Asset cache is invalid and downloads disabled") 115 116 self.log.info("Downloading %s to %s...", self.url, self.cache_file) 117 tmp_cache_file = self.cache_file.with_suffix(".download") 118 119 for retries in range(3): 120 try: 121 with tmp_cache_file.open("xb") as dst: 122 with urllib.request.urlopen(self.url) as resp: 123 copyfileobj(resp, dst) 124 break 125 except FileExistsError: 126 self.log.debug("%s already exists, " 127 "waiting for other thread to finish...", 128 tmp_cache_file) 129 if self._wait_for_other_download(tmp_cache_file): 130 return str(self.cache_file) 131 self.log.debug("%s seems to be stale, " 132 "deleting and retrying download...", 133 tmp_cache_file) 134 tmp_cache_file.unlink() 135 continue 136 except Exception as e: 137 self.log.error("Unable to download %s: %s", self.url, e) 138 tmp_cache_file.unlink() 139 raise 140 141 if not os.path.exists(tmp_cache_file): 142 raise Exception("Retries exceeded downloading %s", self.url) 143 144 try: 145 # Set these just for informational purposes 146 os.setxattr(str(tmp_cache_file), "user.qemu-asset-url", 147 self.url.encode('utf8')) 148 os.setxattr(str(tmp_cache_file), "user.qemu-asset-hash", 149 self.hash.encode('utf8')) 150 except Exception as e: 151 self.log.debug("Unable to set xattr on %s: %s", tmp_cache_file, e) 152 pass 153 154 if not self._check(tmp_cache_file): 155 tmp_cache_file.unlink() 156 raise Exception("Hash of %s does not match %s" % 157 (self.url, self.hash)) 158 tmp_cache_file.replace(self.cache_file) 159 # Remove write perms to stop tests accidentally modifying them 160 os.chmod(self.cache_file, stat.S_IRUSR | stat.S_IRGRP) 161 162 self.log.info("Cached %s at %s" % (self.url, self.cache_file)) 163 return str(self.cache_file) 164 165 def precache_test(test): 166 log = logging.getLogger('qemu-test') 167 log.setLevel(logging.DEBUG) 168 handler = logging.StreamHandler(sys.stdout) 169 handler.setLevel(logging.DEBUG) 170 formatter = logging.Formatter( 171 '%(asctime)s - %(name)s - %(levelname)s - %(message)s') 172 handler.setFormatter(formatter) 173 log.addHandler(handler) 174 for name, asset in vars(test.__class__).items(): 175 if name.startswith("ASSET_") and type(asset) == Asset: 176 log.info("Attempting to cache '%s'" % asset) 177 try: 178 asset.fetch() 179 except HTTPError as e: 180 # Treat 404 as fatal, since it is highly likely to 181 # indicate a broken test rather than a transient 182 # server or networking problem 183 if e.code == 404: 184 raise 185 186 log.debug(f"HTTP error {e.code} from {asset.url} " + 187 "skipping asset precache") 188 189 log.removeHandler(handler) 190 191 def precache_suite(suite): 192 for test in suite: 193 if isinstance(test, unittest.TestSuite): 194 Asset.precache_suite(test) 195 elif isinstance(test, unittest.TestCase): 196 Asset.precache_test(test) 197 198 def precache_suites(path, cacheTstamp): 199 loader = unittest.loader.defaultTestLoader 200 tests = loader.loadTestsFromNames([path], None) 201 202 with open(cacheTstamp, "w") as fh: 203 Asset.precache_suite(tests) 204