14485b04bSFam Zheng#!/usr/bin/env python2 24485b04bSFam Zheng# 34485b04bSFam Zheng# Docker controlling module 44485b04bSFam Zheng# 54485b04bSFam Zheng# Copyright (c) 2016 Red Hat Inc. 64485b04bSFam Zheng# 74485b04bSFam Zheng# Authors: 84485b04bSFam Zheng# Fam Zheng <famz@redhat.com> 94485b04bSFam Zheng# 104485b04bSFam Zheng# This work is licensed under the terms of the GNU GPL, version 2 114485b04bSFam Zheng# or (at your option) any later version. See the COPYING file in 124485b04bSFam Zheng# the top-level directory. 134485b04bSFam Zheng 144485b04bSFam Zhengimport os 154485b04bSFam Zhengimport sys 164485b04bSFam Zhengimport subprocess 174485b04bSFam Zhengimport json 184485b04bSFam Zhengimport hashlib 194485b04bSFam Zhengimport atexit 204485b04bSFam Zhengimport uuid 214485b04bSFam Zhengimport argparse 224485b04bSFam Zhengimport tempfile 23504ca3c2SAlex Bennéeimport re 2497cba1a1SFam Zhengimport signal 256e733da6SAlex Bennéefrom tarfile import TarFile, TarInfo 266e733da6SAlex Bennéefrom StringIO import StringIO 27a9f8d038SAlex Bennéefrom shutil import copy, rmtree 28414a8ce5SAlex Bennéefrom pwd import getpwuid 294485b04bSFam Zheng 30c9772570SSascha Silbe 3106cc3551SPhilippe Mathieu-DaudéFILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy'] 3206cc3551SPhilippe Mathieu-Daudé 3306cc3551SPhilippe Mathieu-Daudé 34c9772570SSascha SilbeDEVNULL = open(os.devnull, 'wb') 35c9772570SSascha Silbe 36c9772570SSascha Silbe 374485b04bSFam Zhengdef _text_checksum(text): 384485b04bSFam Zheng """Calculate a digest string unique to the text content""" 394485b04bSFam Zheng return hashlib.sha1(text).hexdigest() 404485b04bSFam Zheng 414485b04bSFam Zhengdef _guess_docker_command(): 424485b04bSFam Zheng """ Guess a working docker command or raise exception if not found""" 434485b04bSFam Zheng commands = [["docker"], ["sudo", "-n", "docker"]] 444485b04bSFam Zheng for cmd in commands: 450679f98bSEduardo Habkost try: 464485b04bSFam Zheng if subprocess.call(cmd + ["images"], 47c9772570SSascha Silbe stdout=DEVNULL, stderr=DEVNULL) == 0: 484485b04bSFam Zheng return cmd 490679f98bSEduardo Habkost except OSError: 500679f98bSEduardo Habkost pass 514485b04bSFam Zheng commands_txt = "\n".join([" " + " ".join(x) for x in commands]) 524485b04bSFam Zheng raise Exception("Cannot find working docker command. Tried:\n%s" % \ 534485b04bSFam Zheng commands_txt) 544485b04bSFam Zheng 55*2499ee9fSPhilippe Mathieu-Daudédef _copy_with_mkdir(src, root_dir, sub_path='.'): 56504ca3c2SAlex Bennée """Copy src into root_dir, creating sub_path as needed.""" 57504ca3c2SAlex Bennée dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path)) 58504ca3c2SAlex Bennée try: 59504ca3c2SAlex Bennée os.makedirs(dest_dir) 60504ca3c2SAlex Bennée except OSError: 61504ca3c2SAlex Bennée # we can safely ignore already created directories 62504ca3c2SAlex Bennée pass 63504ca3c2SAlex Bennée 64504ca3c2SAlex Bennée dest_file = "%s/%s" % (dest_dir, os.path.basename(src)) 65504ca3c2SAlex Bennée copy(src, dest_file) 66504ca3c2SAlex Bennée 67504ca3c2SAlex Bennée 68504ca3c2SAlex Bennéedef _get_so_libs(executable): 69504ca3c2SAlex Bennée """Return a list of libraries associated with an executable. 70504ca3c2SAlex Bennée 71504ca3c2SAlex Bennée The paths may be symbolic links which would need to be resolved to 72504ca3c2SAlex Bennée ensure theright data is copied.""" 73504ca3c2SAlex Bennée 74504ca3c2SAlex Bennée libs = [] 75504ca3c2SAlex Bennée ldd_re = re.compile(r"(/.*/)(\S*)") 76504ca3c2SAlex Bennée try: 77504ca3c2SAlex Bennée ldd_output = subprocess.check_output(["ldd", executable]) 78504ca3c2SAlex Bennée for line in ldd_output.split("\n"): 79504ca3c2SAlex Bennée search = ldd_re.search(line) 80504ca3c2SAlex Bennée if search and len(search.groups()) == 2: 81504ca3c2SAlex Bennée so_path = search.groups()[0] 82504ca3c2SAlex Bennée so_lib = search.groups()[1] 83504ca3c2SAlex Bennée libs.append("%s/%s" % (so_path, so_lib)) 84504ca3c2SAlex Bennée except subprocess.CalledProcessError: 85504ca3c2SAlex Bennée print "%s had no associated libraries (static build?)" % (executable) 86504ca3c2SAlex Bennée 87504ca3c2SAlex Bennée return libs 88504ca3c2SAlex Bennée 89504ca3c2SAlex Bennéedef _copy_binary_with_libs(src, dest_dir): 90504ca3c2SAlex Bennée """Copy a binary executable and all its dependant libraries. 91504ca3c2SAlex Bennée 92504ca3c2SAlex Bennée This does rely on the host file-system being fairly multi-arch 93504ca3c2SAlex Bennée aware so the file don't clash with the guests layout.""" 94504ca3c2SAlex Bennée 95504ca3c2SAlex Bennée _copy_with_mkdir(src, dest_dir, "/usr/bin") 96504ca3c2SAlex Bennée 97504ca3c2SAlex Bennée libs = _get_so_libs(src) 98504ca3c2SAlex Bennée if libs: 99504ca3c2SAlex Bennée for l in libs: 100504ca3c2SAlex Bennée so_path = os.path.dirname(l) 101504ca3c2SAlex Bennée _copy_with_mkdir(l , dest_dir, so_path) 102504ca3c2SAlex Bennée 1034485b04bSFam Zhengclass Docker(object): 1044485b04bSFam Zheng """ Running Docker commands """ 1054485b04bSFam Zheng def __init__(self): 1064485b04bSFam Zheng self._command = _guess_docker_command() 1074485b04bSFam Zheng self._instances = [] 1084485b04bSFam Zheng atexit.register(self._kill_instances) 10997cba1a1SFam Zheng signal.signal(signal.SIGTERM, self._kill_instances) 11097cba1a1SFam Zheng signal.signal(signal.SIGHUP, self._kill_instances) 1114485b04bSFam Zheng 1126e733da6SAlex Bennée def _do(self, cmd, quiet=True, infile=None, **kwargs): 1134485b04bSFam Zheng if quiet: 114c9772570SSascha Silbe kwargs["stdout"] = DEVNULL 1156e733da6SAlex Bennée if infile: 1166e733da6SAlex Bennée kwargs["stdin"] = infile 1174485b04bSFam Zheng return subprocess.call(self._command + cmd, **kwargs) 1184485b04bSFam Zheng 1194485b04bSFam Zheng def _do_kill_instances(self, only_known, only_active=True): 1204485b04bSFam Zheng cmd = ["ps", "-q"] 1214485b04bSFam Zheng if not only_active: 1224485b04bSFam Zheng cmd.append("-a") 1234485b04bSFam Zheng for i in self._output(cmd).split(): 1244485b04bSFam Zheng resp = self._output(["inspect", i]) 1254485b04bSFam Zheng labels = json.loads(resp)[0]["Config"]["Labels"] 1264485b04bSFam Zheng active = json.loads(resp)[0]["State"]["Running"] 1274485b04bSFam Zheng if not labels: 1284485b04bSFam Zheng continue 1294485b04bSFam Zheng instance_uuid = labels.get("com.qemu.instance.uuid", None) 1304485b04bSFam Zheng if not instance_uuid: 1314485b04bSFam Zheng continue 1324485b04bSFam Zheng if only_known and instance_uuid not in self._instances: 1334485b04bSFam Zheng continue 1344485b04bSFam Zheng print "Terminating", i 1354485b04bSFam Zheng if active: 1364485b04bSFam Zheng self._do(["kill", i]) 1374485b04bSFam Zheng self._do(["rm", i]) 1384485b04bSFam Zheng 1394485b04bSFam Zheng def clean(self): 1404485b04bSFam Zheng self._do_kill_instances(False, False) 1414485b04bSFam Zheng return 0 1424485b04bSFam Zheng 14397cba1a1SFam Zheng def _kill_instances(self, *args, **kwargs): 1444485b04bSFam Zheng return self._do_kill_instances(True) 1454485b04bSFam Zheng 1464485b04bSFam Zheng def _output(self, cmd, **kwargs): 1474485b04bSFam Zheng return subprocess.check_output(self._command + cmd, 1484485b04bSFam Zheng stderr=subprocess.STDOUT, 1494485b04bSFam Zheng **kwargs) 1504485b04bSFam Zheng 1514485b04bSFam Zheng def get_image_dockerfile_checksum(self, tag): 1524485b04bSFam Zheng resp = self._output(["inspect", tag]) 1534485b04bSFam Zheng labels = json.loads(resp)[0]["Config"].get("Labels", {}) 1544485b04bSFam Zheng return labels.get("com.qemu.dockerfile-checksum", "") 1554485b04bSFam Zheng 156414a8ce5SAlex Bennée def build_image(self, tag, docker_dir, dockerfile, 157414a8ce5SAlex Bennée quiet=True, user=False, argv=None): 1584485b04bSFam Zheng if argv == None: 1594485b04bSFam Zheng argv = [] 1604485b04bSFam Zheng 161a9f8d038SAlex Bennée tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker") 1624485b04bSFam Zheng tmp_df.write(dockerfile) 1634485b04bSFam Zheng 164414a8ce5SAlex Bennée if user: 165414a8ce5SAlex Bennée uid = os.getuid() 166414a8ce5SAlex Bennée uname = getpwuid(uid).pw_name 167414a8ce5SAlex Bennée tmp_df.write("\n") 168414a8ce5SAlex Bennée tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" % 169414a8ce5SAlex Bennée (uname, uid, uname)) 170414a8ce5SAlex Bennée 1714485b04bSFam Zheng tmp_df.write("\n") 1724485b04bSFam Zheng tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % 1734485b04bSFam Zheng _text_checksum(dockerfile)) 1744485b04bSFam Zheng tmp_df.flush() 175a9f8d038SAlex Bennée 1764485b04bSFam Zheng self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \ 177a9f8d038SAlex Bennée [docker_dir], 1784485b04bSFam Zheng quiet=quiet) 1794485b04bSFam Zheng 1806e733da6SAlex Bennée def update_image(self, tag, tarball, quiet=True): 1816e733da6SAlex Bennée "Update a tagged image using " 1826e733da6SAlex Bennée 1836e733da6SAlex Bennée self._do(["build", "-t", tag, "-"], quiet=quiet, infile=tarball) 1846e733da6SAlex Bennée 1854485b04bSFam Zheng def image_matches_dockerfile(self, tag, dockerfile): 1864485b04bSFam Zheng try: 1874485b04bSFam Zheng checksum = self.get_image_dockerfile_checksum(tag) 1884485b04bSFam Zheng except Exception: 1894485b04bSFam Zheng return False 1904485b04bSFam Zheng return checksum == _text_checksum(dockerfile) 1914485b04bSFam Zheng 1924485b04bSFam Zheng def run(self, cmd, keep, quiet): 1934485b04bSFam Zheng label = uuid.uuid1().hex 1944485b04bSFam Zheng if not keep: 1954485b04bSFam Zheng self._instances.append(label) 1964485b04bSFam Zheng ret = self._do(["run", "--label", 1974485b04bSFam Zheng "com.qemu.instance.uuid=" + label] + cmd, 1984485b04bSFam Zheng quiet=quiet) 1994485b04bSFam Zheng if not keep: 2004485b04bSFam Zheng self._instances.remove(label) 2014485b04bSFam Zheng return ret 2024485b04bSFam Zheng 2034b08af60SFam Zheng def command(self, cmd, argv, quiet): 2044b08af60SFam Zheng return self._do([cmd] + argv, quiet=quiet) 2054b08af60SFam Zheng 2064485b04bSFam Zhengclass SubCommand(object): 2074485b04bSFam Zheng """A SubCommand template base class""" 2084485b04bSFam Zheng name = None # Subcommand name 2094485b04bSFam Zheng def shared_args(self, parser): 2104485b04bSFam Zheng parser.add_argument("--quiet", action="store_true", 2114485b04bSFam Zheng help="Run quietly unless an error occured") 2124485b04bSFam Zheng 2134485b04bSFam Zheng def args(self, parser): 2144485b04bSFam Zheng """Setup argument parser""" 2154485b04bSFam Zheng pass 2164485b04bSFam Zheng def run(self, args, argv): 2174485b04bSFam Zheng """Run command. 2184485b04bSFam Zheng args: parsed argument by argument parser. 2194485b04bSFam Zheng argv: remaining arguments from sys.argv. 2204485b04bSFam Zheng """ 2214485b04bSFam Zheng pass 2224485b04bSFam Zheng 2234485b04bSFam Zhengclass RunCommand(SubCommand): 2244485b04bSFam Zheng """Invoke docker run and take care of cleaning up""" 2254485b04bSFam Zheng name = "run" 2264485b04bSFam Zheng def args(self, parser): 2274485b04bSFam Zheng parser.add_argument("--keep", action="store_true", 2284485b04bSFam Zheng help="Don't remove image when command completes") 2294485b04bSFam Zheng def run(self, args, argv): 2304485b04bSFam Zheng return Docker().run(argv, args.keep, quiet=args.quiet) 2314485b04bSFam Zheng 2324485b04bSFam Zhengclass BuildCommand(SubCommand): 2334485b04bSFam Zheng """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>""" 2344485b04bSFam Zheng name = "build" 2354485b04bSFam Zheng def args(self, parser): 236504ca3c2SAlex Bennée parser.add_argument("--include-executable", "-e", 237504ca3c2SAlex Bennée help="""Specify a binary that will be copied to the 238504ca3c2SAlex Bennée container together with all its dependent 239504ca3c2SAlex Bennée libraries""") 240414a8ce5SAlex Bennée parser.add_argument("--add-current-user", "-u", dest="user", 241414a8ce5SAlex Bennée action="store_true", 242414a8ce5SAlex Bennée help="Add the current user to image's passwd") 2434485b04bSFam Zheng parser.add_argument("tag", 2444485b04bSFam Zheng help="Image Tag") 2454485b04bSFam Zheng parser.add_argument("dockerfile", 2464485b04bSFam Zheng help="Dockerfile name") 2474485b04bSFam Zheng 2484485b04bSFam Zheng def run(self, args, argv): 2494485b04bSFam Zheng dockerfile = open(args.dockerfile, "rb").read() 2504485b04bSFam Zheng tag = args.tag 2514485b04bSFam Zheng 2524485b04bSFam Zheng dkr = Docker() 2534485b04bSFam Zheng if dkr.image_matches_dockerfile(tag, dockerfile): 2544485b04bSFam Zheng if not args.quiet: 2554485b04bSFam Zheng print "Image is up to date." 256a9f8d038SAlex Bennée else: 257a9f8d038SAlex Bennée # Create a docker context directory for the build 258a9f8d038SAlex Bennée docker_dir = tempfile.mkdtemp(prefix="docker_build") 2594485b04bSFam Zheng 260920776eaSAlex Bennée # Is there a .pre file to run in the build context? 261920776eaSAlex Bennée docker_pre = os.path.splitext(args.dockerfile)[0]+".pre" 262920776eaSAlex Bennée if os.path.exists(docker_pre): 263f8042deaSSascha Silbe stdout = DEVNULL if args.quiet else None 264920776eaSAlex Bennée rc = subprocess.call(os.path.realpath(docker_pre), 265f8042deaSSascha Silbe cwd=docker_dir, stdout=stdout) 266920776eaSAlex Bennée if rc == 3: 267920776eaSAlex Bennée print "Skip" 268920776eaSAlex Bennée return 0 269920776eaSAlex Bennée elif rc != 0: 270920776eaSAlex Bennée print "%s exited with code %d" % (docker_pre, rc) 271920776eaSAlex Bennée return 1 272920776eaSAlex Bennée 273504ca3c2SAlex Bennée # Do we include a extra binary? 274504ca3c2SAlex Bennée if args.include_executable: 275504ca3c2SAlex Bennée _copy_binary_with_libs(args.include_executable, 276504ca3c2SAlex Bennée docker_dir) 277504ca3c2SAlex Bennée 27806cc3551SPhilippe Mathieu-Daudé argv += ["--build-arg=" + k.lower() + "=" + v 27906cc3551SPhilippe Mathieu-Daudé for k, v in os.environ.iteritems() 28006cc3551SPhilippe Mathieu-Daudé if k.lower() in FILTERED_ENV_NAMES] 281a9f8d038SAlex Bennée dkr.build_image(tag, docker_dir, dockerfile, 282414a8ce5SAlex Bennée quiet=args.quiet, user=args.user, argv=argv) 283a9f8d038SAlex Bennée 284a9f8d038SAlex Bennée rmtree(docker_dir) 285a9f8d038SAlex Bennée 2864485b04bSFam Zheng return 0 2874485b04bSFam Zheng 2886e733da6SAlex Bennéeclass UpdateCommand(SubCommand): 2896e733da6SAlex Bennée """ Update a docker image with new executables. Arguments: <tag> <executable>""" 2906e733da6SAlex Bennée name = "update" 2916e733da6SAlex Bennée def args(self, parser): 2926e733da6SAlex Bennée parser.add_argument("tag", 2936e733da6SAlex Bennée help="Image Tag") 2946e733da6SAlex Bennée parser.add_argument("executable", 2956e733da6SAlex Bennée help="Executable to copy") 2966e733da6SAlex Bennée 2976e733da6SAlex Bennée def run(self, args, argv): 2986e733da6SAlex Bennée # Create a temporary tarball with our whole build context and 2996e733da6SAlex Bennée # dockerfile for the update 3006e733da6SAlex Bennée tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz") 3016e733da6SAlex Bennée tmp_tar = TarFile(fileobj=tmp, mode='w') 3026e733da6SAlex Bennée 3036e733da6SAlex Bennée # Add the executable to the tarball 3046e733da6SAlex Bennée bn = os.path.basename(args.executable) 3056e733da6SAlex Bennée ff = "/usr/bin/%s" % bn 3066e733da6SAlex Bennée tmp_tar.add(args.executable, arcname=ff) 3076e733da6SAlex Bennée 3086e733da6SAlex Bennée # Add any associated libraries 3096e733da6SAlex Bennée libs = _get_so_libs(args.executable) 3106e733da6SAlex Bennée if libs: 3116e733da6SAlex Bennée for l in libs: 3126e733da6SAlex Bennée tmp_tar.add(os.path.realpath(l), arcname=l) 3136e733da6SAlex Bennée 3146e733da6SAlex Bennée # Create a Docker buildfile 3156e733da6SAlex Bennée df = StringIO() 3166e733da6SAlex Bennée df.write("FROM %s\n" % args.tag) 3176e733da6SAlex Bennée df.write("ADD . /\n") 3186e733da6SAlex Bennée df.seek(0) 3196e733da6SAlex Bennée 3206e733da6SAlex Bennée df_tar = TarInfo(name="Dockerfile") 3216e733da6SAlex Bennée df_tar.size = len(df.buf) 3226e733da6SAlex Bennée tmp_tar.addfile(df_tar, fileobj=df) 3236e733da6SAlex Bennée 3246e733da6SAlex Bennée tmp_tar.close() 3256e733da6SAlex Bennée 3266e733da6SAlex Bennée # reset the file pointers 3276e733da6SAlex Bennée tmp.flush() 3286e733da6SAlex Bennée tmp.seek(0) 3296e733da6SAlex Bennée 3306e733da6SAlex Bennée # Run the build with our tarball context 3316e733da6SAlex Bennée dkr = Docker() 3326e733da6SAlex Bennée dkr.update_image(args.tag, tmp, quiet=args.quiet) 3336e733da6SAlex Bennée 3346e733da6SAlex Bennée return 0 3356e733da6SAlex Bennée 3364485b04bSFam Zhengclass CleanCommand(SubCommand): 3374485b04bSFam Zheng """Clean up docker instances""" 3384485b04bSFam Zheng name = "clean" 3394485b04bSFam Zheng def run(self, args, argv): 3404485b04bSFam Zheng Docker().clean() 3414485b04bSFam Zheng return 0 3424485b04bSFam Zheng 3434b08af60SFam Zhengclass ImagesCommand(SubCommand): 3444b08af60SFam Zheng """Run "docker images" command""" 3454b08af60SFam Zheng name = "images" 3464b08af60SFam Zheng def run(self, args, argv): 3474b08af60SFam Zheng return Docker().command("images", argv, args.quiet) 3484b08af60SFam Zheng 3494485b04bSFam Zhengdef main(): 3504485b04bSFam Zheng parser = argparse.ArgumentParser(description="A Docker helper", 3514485b04bSFam Zheng usage="%s <subcommand> ..." % os.path.basename(sys.argv[0])) 3524485b04bSFam Zheng subparsers = parser.add_subparsers(title="subcommands", help=None) 3534485b04bSFam Zheng for cls in SubCommand.__subclasses__(): 3544485b04bSFam Zheng cmd = cls() 3554485b04bSFam Zheng subp = subparsers.add_parser(cmd.name, help=cmd.__doc__) 3564485b04bSFam Zheng cmd.shared_args(subp) 3574485b04bSFam Zheng cmd.args(subp) 3584485b04bSFam Zheng subp.set_defaults(cmdobj=cmd) 3594485b04bSFam Zheng args, argv = parser.parse_known_args() 3604485b04bSFam Zheng return args.cmdobj.run(args, argv) 3614485b04bSFam Zheng 3624485b04bSFam Zhengif __name__ == "__main__": 3634485b04bSFam Zheng sys.exit(main()) 364