1# Test utilities for fetching & caching assets 2# 3# Copyright 2024 Red Hat, Inc. 4# 5# This work is licensed under the terms of the GNU GPL, version 2 or 6# later. See the COPYING file in the top-level directory. 7 8import hashlib 9import logging 10import os 11import stat 12import sys 13import unittest 14import urllib.request 15from time import sleep 16from pathlib import Path 17from shutil import copyfileobj 18from urllib.error import HTTPError 19 20 21# Instances of this class must be declared as class level variables 22# starting with a name "ASSET_". This enables the pre-caching logic 23# to easily find all referenced assets and download them prior to 24# execution of the tests. 25class Asset: 26 27 def __init__(self, url, hashsum): 28 self.url = url 29 self.hash = hashsum 30 cache_dir_env = os.getenv('QEMU_TEST_CACHE_DIR') 31 if cache_dir_env: 32 self.cache_dir = Path(cache_dir_env, "download") 33 else: 34 self.cache_dir = Path(Path("~").expanduser(), 35 ".cache", "qemu", "download") 36 self.cache_file = Path(self.cache_dir, hashsum) 37 self.log = logging.getLogger('qemu-test') 38 39 def __repr__(self): 40 return "Asset: url=%s hash=%s cache=%s" % ( 41 self.url, self.hash, self.cache_file) 42 43 def __str__(self): 44 return str(self.cache_file) 45 46 def _check(self, cache_file): 47 if self.hash is None: 48 return True 49 if len(self.hash) == 64: 50 hl = hashlib.sha256() 51 elif len(self.hash) == 128: 52 hl = hashlib.sha512() 53 else: 54 raise Exception("unknown hash type") 55 56 # Calculate the hash of the file: 57 with open(cache_file, 'rb') as file: 58 while True: 59 chunk = file.read(1 << 20) 60 if not chunk: 61 break 62 hl.update(chunk) 63 64 return self.hash == hl.hexdigest() 65 66 def valid(self): 67 return self.cache_file.exists() and self._check(self.cache_file) 68 69 def fetchable(self): 70 return not os.environ.get("QEMU_TEST_NO_DOWNLOAD", False) 71 72 def available(self): 73 return self.valid() or self.fetchable() 74 75 def _wait_for_other_download(self, tmp_cache_file): 76 # Another thread already seems to download the asset, so wait until 77 # it is done, while also checking the size to see whether it is stuck 78 try: 79 current_size = tmp_cache_file.stat().st_size 80 new_size = current_size 81 except: 82 if os.path.exists(self.cache_file): 83 return True 84 raise 85 waittime = lastchange = 600 86 while waittime > 0: 87 sleep(1) 88 waittime -= 1 89 try: 90 new_size = tmp_cache_file.stat().st_size 91 except: 92 if os.path.exists(self.cache_file): 93 return True 94 raise 95 if new_size != current_size: 96 lastchange = waittime 97 current_size = new_size 98 elif lastchange - waittime > 90: 99 return False 100 101 self.log.debug("Time out while waiting for %s!", tmp_cache_file) 102 raise 103 104 def fetch(self): 105 if not self.cache_dir.exists(): 106 self.cache_dir.mkdir(parents=True, exist_ok=True) 107 108 if self.valid(): 109 self.log.debug("Using cached asset %s for %s", 110 self.cache_file, self.url) 111 return str(self.cache_file) 112 113 if not self.fetchable(): 114 raise Exception("Asset cache is invalid and downloads disabled") 115 116 self.log.info("Downloading %s to %s...", self.url, self.cache_file) 117 tmp_cache_file = self.cache_file.with_suffix(".download") 118 119 for retries in range(3): 120 try: 121 with tmp_cache_file.open("xb") as dst: 122 with urllib.request.urlopen(self.url) as resp: 123 copyfileobj(resp, dst) 124 break 125 except FileExistsError: 126 self.log.debug("%s already exists, " 127 "waiting for other thread to finish...", 128 tmp_cache_file) 129 if self._wait_for_other_download(tmp_cache_file): 130 return str(self.cache_file) 131 self.log.debug("%s seems to be stale, " 132 "deleting and retrying download...", 133 tmp_cache_file) 134 tmp_cache_file.unlink() 135 continue 136 except Exception as e: 137 self.log.error("Unable to download %s: %s", self.url, e) 138 tmp_cache_file.unlink() 139 raise 140 141 try: 142 # Set these just for informational purposes 143 os.setxattr(str(tmp_cache_file), "user.qemu-asset-url", 144 self.url.encode('utf8')) 145 os.setxattr(str(tmp_cache_file), "user.qemu-asset-hash", 146 self.hash.encode('utf8')) 147 except Exception as e: 148 self.log.debug("Unable to set xattr on %s: %s", tmp_cache_file, e) 149 pass 150 151 if not self._check(tmp_cache_file): 152 tmp_cache_file.unlink() 153 raise Exception("Hash of %s does not match %s" % 154 (self.url, self.hash)) 155 tmp_cache_file.replace(self.cache_file) 156 # Remove write perms to stop tests accidentally modifying them 157 os.chmod(self.cache_file, stat.S_IRUSR | stat.S_IRGRP) 158 159 self.log.info("Cached %s at %s" % (self.url, self.cache_file)) 160 return str(self.cache_file) 161 162 def precache_test(test): 163 log = logging.getLogger('qemu-test') 164 log.setLevel(logging.DEBUG) 165 handler = logging.StreamHandler(sys.stdout) 166 handler.setLevel(logging.DEBUG) 167 formatter = logging.Formatter( 168 '%(asctime)s - %(name)s - %(levelname)s - %(message)s') 169 handler.setFormatter(formatter) 170 log.addHandler(handler) 171 for name, asset in vars(test.__class__).items(): 172 if name.startswith("ASSET_") and type(asset) == Asset: 173 log.info("Attempting to cache '%s'" % asset) 174 try: 175 asset.fetch() 176 except HTTPError as e: 177 # Treat 404 as fatal, since it is highly likely to 178 # indicate a broken test rather than a transient 179 # server or networking problem 180 if e.code == 404: 181 raise 182 183 log.debug(f"HTTP error {e.code} from {asset.url} " + 184 "skipping asset precache") 185 186 log.removeHandler(handler) 187 188 def precache_suite(suite): 189 for test in suite: 190 if isinstance(test, unittest.TestSuite): 191 Asset.precache_suite(test) 192 elif isinstance(test, unittest.TestCase): 193 Asset.precache_test(test) 194 195 def precache_suites(path, cacheTstamp): 196 loader = unittest.loader.defaultTestLoader 197 tests = loader.loadTestsFromNames([path], None) 198 199 with open(cacheTstamp, "w") as fh: 200 Asset.precache_suite(tests) 201