xref: /qemu/tests/docker/docker.py (revision 2499ee9fada75a9581b0bbe8ffc7637c7db2f306)
14485b04bSFam Zheng#!/usr/bin/env python2
24485b04bSFam Zheng#
34485b04bSFam Zheng# Docker controlling module
44485b04bSFam Zheng#
54485b04bSFam Zheng# Copyright (c) 2016 Red Hat Inc.
64485b04bSFam Zheng#
74485b04bSFam Zheng# Authors:
84485b04bSFam Zheng#  Fam Zheng <famz@redhat.com>
94485b04bSFam Zheng#
104485b04bSFam Zheng# This work is licensed under the terms of the GNU GPL, version 2
114485b04bSFam Zheng# or (at your option) any later version. See the COPYING file in
124485b04bSFam Zheng# the top-level directory.
134485b04bSFam Zheng
144485b04bSFam Zhengimport os
154485b04bSFam Zhengimport sys
164485b04bSFam Zhengimport subprocess
174485b04bSFam Zhengimport json
184485b04bSFam Zhengimport hashlib
194485b04bSFam Zhengimport atexit
204485b04bSFam Zhengimport uuid
214485b04bSFam Zhengimport argparse
224485b04bSFam Zhengimport tempfile
23504ca3c2SAlex Bennéeimport re
2497cba1a1SFam Zhengimport signal
256e733da6SAlex Bennéefrom tarfile import TarFile, TarInfo
266e733da6SAlex Bennéefrom StringIO import StringIO
27a9f8d038SAlex Bennéefrom shutil import copy, rmtree
28414a8ce5SAlex Bennéefrom pwd import getpwuid
294485b04bSFam Zheng
30c9772570SSascha Silbe
3106cc3551SPhilippe Mathieu-DaudéFILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
3206cc3551SPhilippe Mathieu-Daudé
3306cc3551SPhilippe Mathieu-Daudé
34c9772570SSascha SilbeDEVNULL = open(os.devnull, 'wb')
35c9772570SSascha Silbe
36c9772570SSascha Silbe
374485b04bSFam Zhengdef _text_checksum(text):
384485b04bSFam Zheng    """Calculate a digest string unique to the text content"""
394485b04bSFam Zheng    return hashlib.sha1(text).hexdigest()
404485b04bSFam Zheng
414485b04bSFam Zhengdef _guess_docker_command():
424485b04bSFam Zheng    """ Guess a working docker command or raise exception if not found"""
434485b04bSFam Zheng    commands = [["docker"], ["sudo", "-n", "docker"]]
444485b04bSFam Zheng    for cmd in commands:
450679f98bSEduardo Habkost        try:
464485b04bSFam Zheng            if subprocess.call(cmd + ["images"],
47c9772570SSascha Silbe                               stdout=DEVNULL, stderr=DEVNULL) == 0:
484485b04bSFam Zheng                return cmd
490679f98bSEduardo Habkost        except OSError:
500679f98bSEduardo Habkost            pass
514485b04bSFam Zheng    commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
524485b04bSFam Zheng    raise Exception("Cannot find working docker command. Tried:\n%s" % \
534485b04bSFam Zheng                    commands_txt)
544485b04bSFam Zheng
55*2499ee9fSPhilippe Mathieu-Daudédef _copy_with_mkdir(src, root_dir, sub_path='.'):
56504ca3c2SAlex Bennée    """Copy src into root_dir, creating sub_path as needed."""
57504ca3c2SAlex Bennée    dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
58504ca3c2SAlex Bennée    try:
59504ca3c2SAlex Bennée        os.makedirs(dest_dir)
60504ca3c2SAlex Bennée    except OSError:
61504ca3c2SAlex Bennée        # we can safely ignore already created directories
62504ca3c2SAlex Bennée        pass
63504ca3c2SAlex Bennée
64504ca3c2SAlex Bennée    dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
65504ca3c2SAlex Bennée    copy(src, dest_file)
66504ca3c2SAlex Bennée
67504ca3c2SAlex Bennée
68504ca3c2SAlex Bennéedef _get_so_libs(executable):
69504ca3c2SAlex Bennée    """Return a list of libraries associated with an executable.
70504ca3c2SAlex Bennée
71504ca3c2SAlex Bennée    The paths may be symbolic links which would need to be resolved to
72504ca3c2SAlex Bennée    ensure theright data is copied."""
73504ca3c2SAlex Bennée
74504ca3c2SAlex Bennée    libs = []
75504ca3c2SAlex Bennée    ldd_re = re.compile(r"(/.*/)(\S*)")
76504ca3c2SAlex Bennée    try:
77504ca3c2SAlex Bennée        ldd_output = subprocess.check_output(["ldd", executable])
78504ca3c2SAlex Bennée        for line in ldd_output.split("\n"):
79504ca3c2SAlex Bennée            search = ldd_re.search(line)
80504ca3c2SAlex Bennée            if search and len(search.groups()) == 2:
81504ca3c2SAlex Bennée                so_path = search.groups()[0]
82504ca3c2SAlex Bennée                so_lib = search.groups()[1]
83504ca3c2SAlex Bennée                libs.append("%s/%s" % (so_path, so_lib))
84504ca3c2SAlex Bennée    except subprocess.CalledProcessError:
85504ca3c2SAlex Bennée        print "%s had no associated libraries (static build?)" % (executable)
86504ca3c2SAlex Bennée
87504ca3c2SAlex Bennée    return libs
88504ca3c2SAlex Bennée
89504ca3c2SAlex Bennéedef _copy_binary_with_libs(src, dest_dir):
90504ca3c2SAlex Bennée    """Copy a binary executable and all its dependant libraries.
91504ca3c2SAlex Bennée
92504ca3c2SAlex Bennée    This does rely on the host file-system being fairly multi-arch
93504ca3c2SAlex Bennée    aware so the file don't clash with the guests layout."""
94504ca3c2SAlex Bennée
95504ca3c2SAlex Bennée    _copy_with_mkdir(src, dest_dir, "/usr/bin")
96504ca3c2SAlex Bennée
97504ca3c2SAlex Bennée    libs = _get_so_libs(src)
98504ca3c2SAlex Bennée    if libs:
99504ca3c2SAlex Bennée        for l in libs:
100504ca3c2SAlex Bennée            so_path = os.path.dirname(l)
101504ca3c2SAlex Bennée            _copy_with_mkdir(l , dest_dir, so_path)
102504ca3c2SAlex Bennée
1034485b04bSFam Zhengclass Docker(object):
1044485b04bSFam Zheng    """ Running Docker commands """
1054485b04bSFam Zheng    def __init__(self):
1064485b04bSFam Zheng        self._command = _guess_docker_command()
1074485b04bSFam Zheng        self._instances = []
1084485b04bSFam Zheng        atexit.register(self._kill_instances)
10997cba1a1SFam Zheng        signal.signal(signal.SIGTERM, self._kill_instances)
11097cba1a1SFam Zheng        signal.signal(signal.SIGHUP, self._kill_instances)
1114485b04bSFam Zheng
1126e733da6SAlex Bennée    def _do(self, cmd, quiet=True, infile=None, **kwargs):
1134485b04bSFam Zheng        if quiet:
114c9772570SSascha Silbe            kwargs["stdout"] = DEVNULL
1156e733da6SAlex Bennée        if infile:
1166e733da6SAlex Bennée            kwargs["stdin"] = infile
1174485b04bSFam Zheng        return subprocess.call(self._command + cmd, **kwargs)
1184485b04bSFam Zheng
1194485b04bSFam Zheng    def _do_kill_instances(self, only_known, only_active=True):
1204485b04bSFam Zheng        cmd = ["ps", "-q"]
1214485b04bSFam Zheng        if not only_active:
1224485b04bSFam Zheng            cmd.append("-a")
1234485b04bSFam Zheng        for i in self._output(cmd).split():
1244485b04bSFam Zheng            resp = self._output(["inspect", i])
1254485b04bSFam Zheng            labels = json.loads(resp)[0]["Config"]["Labels"]
1264485b04bSFam Zheng            active = json.loads(resp)[0]["State"]["Running"]
1274485b04bSFam Zheng            if not labels:
1284485b04bSFam Zheng                continue
1294485b04bSFam Zheng            instance_uuid = labels.get("com.qemu.instance.uuid", None)
1304485b04bSFam Zheng            if not instance_uuid:
1314485b04bSFam Zheng                continue
1324485b04bSFam Zheng            if only_known and instance_uuid not in self._instances:
1334485b04bSFam Zheng                continue
1344485b04bSFam Zheng            print "Terminating", i
1354485b04bSFam Zheng            if active:
1364485b04bSFam Zheng                self._do(["kill", i])
1374485b04bSFam Zheng            self._do(["rm", i])
1384485b04bSFam Zheng
1394485b04bSFam Zheng    def clean(self):
1404485b04bSFam Zheng        self._do_kill_instances(False, False)
1414485b04bSFam Zheng        return 0
1424485b04bSFam Zheng
14397cba1a1SFam Zheng    def _kill_instances(self, *args, **kwargs):
1444485b04bSFam Zheng        return self._do_kill_instances(True)
1454485b04bSFam Zheng
1464485b04bSFam Zheng    def _output(self, cmd, **kwargs):
1474485b04bSFam Zheng        return subprocess.check_output(self._command + cmd,
1484485b04bSFam Zheng                                       stderr=subprocess.STDOUT,
1494485b04bSFam Zheng                                       **kwargs)
1504485b04bSFam Zheng
1514485b04bSFam Zheng    def get_image_dockerfile_checksum(self, tag):
1524485b04bSFam Zheng        resp = self._output(["inspect", tag])
1534485b04bSFam Zheng        labels = json.loads(resp)[0]["Config"].get("Labels", {})
1544485b04bSFam Zheng        return labels.get("com.qemu.dockerfile-checksum", "")
1554485b04bSFam Zheng
156414a8ce5SAlex Bennée    def build_image(self, tag, docker_dir, dockerfile,
157414a8ce5SAlex Bennée                    quiet=True, user=False, argv=None):
1584485b04bSFam Zheng        if argv == None:
1594485b04bSFam Zheng            argv = []
1604485b04bSFam Zheng
161a9f8d038SAlex Bennée        tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
1624485b04bSFam Zheng        tmp_df.write(dockerfile)
1634485b04bSFam Zheng
164414a8ce5SAlex Bennée        if user:
165414a8ce5SAlex Bennée            uid = os.getuid()
166414a8ce5SAlex Bennée            uname = getpwuid(uid).pw_name
167414a8ce5SAlex Bennée            tmp_df.write("\n")
168414a8ce5SAlex Bennée            tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
169414a8ce5SAlex Bennée                         (uname, uid, uname))
170414a8ce5SAlex Bennée
1714485b04bSFam Zheng        tmp_df.write("\n")
1724485b04bSFam Zheng        tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
1734485b04bSFam Zheng                     _text_checksum(dockerfile))
1744485b04bSFam Zheng        tmp_df.flush()
175a9f8d038SAlex Bennée
1764485b04bSFam Zheng        self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \
177a9f8d038SAlex Bennée                 [docker_dir],
1784485b04bSFam Zheng                 quiet=quiet)
1794485b04bSFam Zheng
1806e733da6SAlex Bennée    def update_image(self, tag, tarball, quiet=True):
1816e733da6SAlex Bennée        "Update a tagged image using "
1826e733da6SAlex Bennée
1836e733da6SAlex Bennée        self._do(["build", "-t", tag, "-"], quiet=quiet, infile=tarball)
1846e733da6SAlex Bennée
1854485b04bSFam Zheng    def image_matches_dockerfile(self, tag, dockerfile):
1864485b04bSFam Zheng        try:
1874485b04bSFam Zheng            checksum = self.get_image_dockerfile_checksum(tag)
1884485b04bSFam Zheng        except Exception:
1894485b04bSFam Zheng            return False
1904485b04bSFam Zheng        return checksum == _text_checksum(dockerfile)
1914485b04bSFam Zheng
1924485b04bSFam Zheng    def run(self, cmd, keep, quiet):
1934485b04bSFam Zheng        label = uuid.uuid1().hex
1944485b04bSFam Zheng        if not keep:
1954485b04bSFam Zheng            self._instances.append(label)
1964485b04bSFam Zheng        ret = self._do(["run", "--label",
1974485b04bSFam Zheng                        "com.qemu.instance.uuid=" + label] + cmd,
1984485b04bSFam Zheng                       quiet=quiet)
1994485b04bSFam Zheng        if not keep:
2004485b04bSFam Zheng            self._instances.remove(label)
2014485b04bSFam Zheng        return ret
2024485b04bSFam Zheng
2034b08af60SFam Zheng    def command(self, cmd, argv, quiet):
2044b08af60SFam Zheng        return self._do([cmd] + argv, quiet=quiet)
2054b08af60SFam Zheng
2064485b04bSFam Zhengclass SubCommand(object):
2074485b04bSFam Zheng    """A SubCommand template base class"""
2084485b04bSFam Zheng    name = None # Subcommand name
2094485b04bSFam Zheng    def shared_args(self, parser):
2104485b04bSFam Zheng        parser.add_argument("--quiet", action="store_true",
2114485b04bSFam Zheng                            help="Run quietly unless an error occured")
2124485b04bSFam Zheng
2134485b04bSFam Zheng    def args(self, parser):
2144485b04bSFam Zheng        """Setup argument parser"""
2154485b04bSFam Zheng        pass
2164485b04bSFam Zheng    def run(self, args, argv):
2174485b04bSFam Zheng        """Run command.
2184485b04bSFam Zheng        args: parsed argument by argument parser.
2194485b04bSFam Zheng        argv: remaining arguments from sys.argv.
2204485b04bSFam Zheng        """
2214485b04bSFam Zheng        pass
2224485b04bSFam Zheng
2234485b04bSFam Zhengclass RunCommand(SubCommand):
2244485b04bSFam Zheng    """Invoke docker run and take care of cleaning up"""
2254485b04bSFam Zheng    name = "run"
2264485b04bSFam Zheng    def args(self, parser):
2274485b04bSFam Zheng        parser.add_argument("--keep", action="store_true",
2284485b04bSFam Zheng                            help="Don't remove image when command completes")
2294485b04bSFam Zheng    def run(self, args, argv):
2304485b04bSFam Zheng        return Docker().run(argv, args.keep, quiet=args.quiet)
2314485b04bSFam Zheng
2324485b04bSFam Zhengclass BuildCommand(SubCommand):
2334485b04bSFam Zheng    """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
2344485b04bSFam Zheng    name = "build"
2354485b04bSFam Zheng    def args(self, parser):
236504ca3c2SAlex Bennée        parser.add_argument("--include-executable", "-e",
237504ca3c2SAlex Bennée                            help="""Specify a binary that will be copied to the
238504ca3c2SAlex Bennée                            container together with all its dependent
239504ca3c2SAlex Bennée                            libraries""")
240414a8ce5SAlex Bennée        parser.add_argument("--add-current-user", "-u", dest="user",
241414a8ce5SAlex Bennée                            action="store_true",
242414a8ce5SAlex Bennée                            help="Add the current user to image's passwd")
2434485b04bSFam Zheng        parser.add_argument("tag",
2444485b04bSFam Zheng                            help="Image Tag")
2454485b04bSFam Zheng        parser.add_argument("dockerfile",
2464485b04bSFam Zheng                            help="Dockerfile name")
2474485b04bSFam Zheng
2484485b04bSFam Zheng    def run(self, args, argv):
2494485b04bSFam Zheng        dockerfile = open(args.dockerfile, "rb").read()
2504485b04bSFam Zheng        tag = args.tag
2514485b04bSFam Zheng
2524485b04bSFam Zheng        dkr = Docker()
2534485b04bSFam Zheng        if dkr.image_matches_dockerfile(tag, dockerfile):
2544485b04bSFam Zheng            if not args.quiet:
2554485b04bSFam Zheng                print "Image is up to date."
256a9f8d038SAlex Bennée        else:
257a9f8d038SAlex Bennée            # Create a docker context directory for the build
258a9f8d038SAlex Bennée            docker_dir = tempfile.mkdtemp(prefix="docker_build")
2594485b04bSFam Zheng
260920776eaSAlex Bennée            # Is there a .pre file to run in the build context?
261920776eaSAlex Bennée            docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
262920776eaSAlex Bennée            if os.path.exists(docker_pre):
263f8042deaSSascha Silbe                stdout = DEVNULL if args.quiet else None
264920776eaSAlex Bennée                rc = subprocess.call(os.path.realpath(docker_pre),
265f8042deaSSascha Silbe                                     cwd=docker_dir, stdout=stdout)
266920776eaSAlex Bennée                if rc == 3:
267920776eaSAlex Bennée                    print "Skip"
268920776eaSAlex Bennée                    return 0
269920776eaSAlex Bennée                elif rc != 0:
270920776eaSAlex Bennée                    print "%s exited with code %d" % (docker_pre, rc)
271920776eaSAlex Bennée                    return 1
272920776eaSAlex Bennée
273504ca3c2SAlex Bennée            # Do we include a extra binary?
274504ca3c2SAlex Bennée            if args.include_executable:
275504ca3c2SAlex Bennée                _copy_binary_with_libs(args.include_executable,
276504ca3c2SAlex Bennée                                       docker_dir)
277504ca3c2SAlex Bennée
27806cc3551SPhilippe Mathieu-Daudé            argv += ["--build-arg=" + k.lower() + "=" + v
27906cc3551SPhilippe Mathieu-Daudé                        for k, v in os.environ.iteritems()
28006cc3551SPhilippe Mathieu-Daudé                        if k.lower() in FILTERED_ENV_NAMES]
281a9f8d038SAlex Bennée            dkr.build_image(tag, docker_dir, dockerfile,
282414a8ce5SAlex Bennée                            quiet=args.quiet, user=args.user, argv=argv)
283a9f8d038SAlex Bennée
284a9f8d038SAlex Bennée            rmtree(docker_dir)
285a9f8d038SAlex Bennée
2864485b04bSFam Zheng        return 0
2874485b04bSFam Zheng
2886e733da6SAlex Bennéeclass UpdateCommand(SubCommand):
2896e733da6SAlex Bennée    """ Update a docker image with new executables. Arguments: <tag> <executable>"""
2906e733da6SAlex Bennée    name = "update"
2916e733da6SAlex Bennée    def args(self, parser):
2926e733da6SAlex Bennée        parser.add_argument("tag",
2936e733da6SAlex Bennée                            help="Image Tag")
2946e733da6SAlex Bennée        parser.add_argument("executable",
2956e733da6SAlex Bennée                            help="Executable to copy")
2966e733da6SAlex Bennée
2976e733da6SAlex Bennée    def run(self, args, argv):
2986e733da6SAlex Bennée        # Create a temporary tarball with our whole build context and
2996e733da6SAlex Bennée        # dockerfile for the update
3006e733da6SAlex Bennée        tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
3016e733da6SAlex Bennée        tmp_tar = TarFile(fileobj=tmp, mode='w')
3026e733da6SAlex Bennée
3036e733da6SAlex Bennée        # Add the executable to the tarball
3046e733da6SAlex Bennée        bn = os.path.basename(args.executable)
3056e733da6SAlex Bennée        ff = "/usr/bin/%s" % bn
3066e733da6SAlex Bennée        tmp_tar.add(args.executable, arcname=ff)
3076e733da6SAlex Bennée
3086e733da6SAlex Bennée        # Add any associated libraries
3096e733da6SAlex Bennée        libs = _get_so_libs(args.executable)
3106e733da6SAlex Bennée        if libs:
3116e733da6SAlex Bennée            for l in libs:
3126e733da6SAlex Bennée                tmp_tar.add(os.path.realpath(l), arcname=l)
3136e733da6SAlex Bennée
3146e733da6SAlex Bennée        # Create a Docker buildfile
3156e733da6SAlex Bennée        df = StringIO()
3166e733da6SAlex Bennée        df.write("FROM %s\n" % args.tag)
3176e733da6SAlex Bennée        df.write("ADD . /\n")
3186e733da6SAlex Bennée        df.seek(0)
3196e733da6SAlex Bennée
3206e733da6SAlex Bennée        df_tar = TarInfo(name="Dockerfile")
3216e733da6SAlex Bennée        df_tar.size = len(df.buf)
3226e733da6SAlex Bennée        tmp_tar.addfile(df_tar, fileobj=df)
3236e733da6SAlex Bennée
3246e733da6SAlex Bennée        tmp_tar.close()
3256e733da6SAlex Bennée
3266e733da6SAlex Bennée        # reset the file pointers
3276e733da6SAlex Bennée        tmp.flush()
3286e733da6SAlex Bennée        tmp.seek(0)
3296e733da6SAlex Bennée
3306e733da6SAlex Bennée        # Run the build with our tarball context
3316e733da6SAlex Bennée        dkr = Docker()
3326e733da6SAlex Bennée        dkr.update_image(args.tag, tmp, quiet=args.quiet)
3336e733da6SAlex Bennée
3346e733da6SAlex Bennée        return 0
3356e733da6SAlex Bennée
3364485b04bSFam Zhengclass CleanCommand(SubCommand):
3374485b04bSFam Zheng    """Clean up docker instances"""
3384485b04bSFam Zheng    name = "clean"
3394485b04bSFam Zheng    def run(self, args, argv):
3404485b04bSFam Zheng        Docker().clean()
3414485b04bSFam Zheng        return 0
3424485b04bSFam Zheng
3434b08af60SFam Zhengclass ImagesCommand(SubCommand):
3444b08af60SFam Zheng    """Run "docker images" command"""
3454b08af60SFam Zheng    name = "images"
3464b08af60SFam Zheng    def run(self, args, argv):
3474b08af60SFam Zheng        return Docker().command("images", argv, args.quiet)
3484b08af60SFam Zheng
3494485b04bSFam Zhengdef main():
3504485b04bSFam Zheng    parser = argparse.ArgumentParser(description="A Docker helper",
3514485b04bSFam Zheng            usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
3524485b04bSFam Zheng    subparsers = parser.add_subparsers(title="subcommands", help=None)
3534485b04bSFam Zheng    for cls in SubCommand.__subclasses__():
3544485b04bSFam Zheng        cmd = cls()
3554485b04bSFam Zheng        subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
3564485b04bSFam Zheng        cmd.shared_args(subp)
3574485b04bSFam Zheng        cmd.args(subp)
3584485b04bSFam Zheng        subp.set_defaults(cmdobj=cmd)
3594485b04bSFam Zheng    args, argv = parser.parse_known_args()
3604485b04bSFam Zheng    return args.cmdobj.run(args, argv)
3614485b04bSFam Zheng
3624485b04bSFam Zhengif __name__ == "__main__":
3634485b04bSFam Zheng    sys.exit(main())
364