#!/usr/bin/python3

# this testsuite is part of autopkgtest
# autopkgtest is a tool for testing Debian binary packages
#
# autopkgtest is Copyright (C) 2006-2016 Canonical Ltd.
# Author: Martin Pitt <martin.pitt@ubuntu.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# See the file CREDITS for a full list of credits information (often
# installed as /usr/share/doc/autopkgtest/CREDITS).

import sys
import os
import re
import subprocess
import unittest
import tempfile
import shutil
import fnmatch
import time
import json
from glob import glob

test_dir = os.path.dirname(os.path.abspath(__file__))
root_dir = os.path.dirname(test_dir)

sys.path.insert(1, test_dir)
import testarchive

# in some corner cases apt-get download might not be available in a build
# environment, so check if this actually works
try:
    have_apt = subprocess.call(['apt-get', 'download', 'gir1.2-json-1.0'],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               cwd=os.environ.get('TMPDIR', '/tmp')) == 0
except OSError:
    have_apt = False

try:
    have_apt_src = subprocess.call(['apt-cache', 'showsrc', 'coreutils'],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT) == 0
except OSError:
    have_apt_src = False

have_ubuntu_device_flash = subprocess.call(['sh', '-ec', 'command -v ubuntu-device-flash'],
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE) == 0

have_autodep8 = subprocess.call(['sh', '-ec', 'command -v autodep8'],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE) == 0
have_git = subprocess.call(['sh', '-ec', 'command -v git'],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE) == 0

host_arch = subprocess.run(['dpkg-architecture', '-q', 'DEB_HOST_ARCH'],
                           stdout=subprocess.PIPE, universal_newlines=True).stdout.strip()

host_os = subprocess.run(['dpkg-architecture', '-q', 'DEB_HOST_ARCH_OS'],
                         stdout=subprocess.PIPE, universal_newlines=True).stdout.strip()


class AdtTestCase(unittest.TestCase):
    '''Base class with common test setup'''

    def __init__(self, virt_args, *args, **kwargs):
        super(AdtTestCase, self).__init__(*args, **kwargs)
        self.exe = os.path.join(root_dir, 'runner', os.path.basename(__file__))
        self.virt_args = virt_args
        self.orig_home = os.path.expanduser('~')

    def setUp(self):
        self.workdir = tempfile.mkdtemp(prefix='autopkgtest.test.')
        os.chmod(self.workdir, 0o755)
        self.addCleanup(shutil.rmtree, self.workdir)
        self.cwd = os.getcwd()
        self.addCleanup(os.chdir, self.cwd)

        temp_home = os.path.join(self.workdir, 'home')
        shutil.copytree(os.path.join(test_dir, 'home'), temp_home)
        os.chmod(os.path.join(temp_home, '.ssh', 'id_rsa'), 0o600)
        os.mkdir(os.path.join(temp_home, '.cache'))

        # avoid re-downloading ubuntu-device-flash images
        os.symlink(os.path.join(self.orig_home, '.cache', 'ubuntuimages'),
                   os.path.join(temp_home, '.cache', 'ubuntuimages'))

        # also keep per-user LXC containers
        lxc_orig = os.path.join(self.orig_home, '.local', 'share', 'lxc')
        if os.path.isdir(lxc_orig):
            lxc_temp = os.path.join(temp_home, '.local', 'share', 'lxc')
            os.makedirs(os.path.dirname(lxc_temp))
            os.symlink(lxc_orig, lxc_temp)

        # keep LXD client certs and config
        lxc_orig = os.path.join(self.orig_home, '.config', 'lxc')
        lxc_temp = os.path.join(temp_home, '.config', 'lxc')
        os.makedirs(os.path.dirname(lxc_temp))
        os.symlink(lxc_orig, lxc_temp)

        os.environ['HOME'] = temp_home

    def build_src(self, test_control, test_scripts, pkgname='testpkg'):
        '''Create source package tree with given tests.

         @test_control: contents of debian/tests/control
         @test_scripts: map of test name (in debian/tests/) to file contents

        Return path to the source tree.
        '''
        srcdir = os.path.join(self.workdir, pkgname)
        shutil.copytree(os.path.join(test_dir, 'testpkg'), srcdir, symlinks=True)
        if test_control:
            dtdir = os.path.join(srcdir, 'debian', 'tests')
            os.mkdir(dtdir)
            with open(os.path.join(dtdir, 'control'), 'w', encoding='UTF-8') as f:
                f.write(test_control)
            for name, contents in test_scripts.items():
                with open(os.path.join(dtdir, name), 'w', encoding='UTF-8') as f:
                    f.write(contents)

        return srcdir

    def build_dsc(self, test_control, test_scripts):
        '''Create source package dsc with given tests.

         @test_control: contents of debian/tests/control
         @test_scripts: map of test name (in debian/tests/) to file contents

        Return path to the dsc.
        '''
        srcdir = self.build_src(test_control, test_scripts)
        dbp = subprocess.Popen(['dpkg-buildpackage', '-S', '-us', '-uc'],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               cwd=srcdir)
        out, err = dbp.communicate()
        self.assertEqual(dbp.returncode, 0, err)
        return os.path.join(os.path.dirname(srcdir), 'testpkg_1.dsc')

    def runtest(self, args, virt_args=None, env=None):
        '''Run autopkgtest with given arguments with configured virt runner.

         @args: command line args of autopkgtest, excluding the command name
                itself; "-- virt" will be appended automatically
                (called from the source tree)

        Return a tuple (exit_code, stdout, stderr).
        '''
        p = subprocess.Popen([self.exe] + args +
                             ['--'] + (virt_args or self.virt_args),
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                             env=env)
        (out, err) = p.communicate()
        return (p.returncode, out.decode('UTF-8', 'replace'), err.decode('UTF-8', 'replace'))

    @property
    def has_isolation_machine(self):
        return self.virt_args[0] in ['qemu', 'null']

    @property
    def has_isolation_container(self):
        return self.virt_args[0] not in ['chroot', 'schroot']

    @property
    def can_revert_full_system(self):
        return self.virt_args[0] not in ['null', 'chroot', 'schroot']


class DebTestsAll:
    '''Common deb tests for all runners'''

    def test_tree_norestrictions_nobuild_success(self):
        '''source tree, no build, no restrictions, test success'''

        p = self.build_src('Tests: pass\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        # print('----- out ----\n%s\n----- err ----\n%s\n----' % (out, err))
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        self.assertIn('processing dependency coreutils', err)
        self.assertNotIn('@@@@@@ test bed setup', err)
        # should show test stdout
        self.assertRegex(out, r'(^|\n)I am fine\n')
        # should show summary
        self.assertRegex(err, r'@@@ summary\npass\s+PASS\nautopkgtest: DBG')
        # should log kernel version
        self.assertRegex(err, r'testbed running kernel: Linux \d')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)
        # should not build package
        self.assertNotIn('dh build', err)
        # should log package version
        self.assertIn('testing package testpkg version 1\n', err)

    @unittest.skipIf((os.getuid() == 0 and
                      os.path.exists('/usr/share/doc/aspell-doc')),
                     'needs aspell-doc uninstalled if run as root')
    def test_tree_build_needed_success(self):
        '''source tree, build-needed restriction, test success'''

        p = self.build_src('Tests: pass\nDepends: coreutils\nRestrictions: build-needed\n',
                           {'pass': '#!/bin/sh -e\n./test_built | grep -q "built script OK"\n'
                            './test_abspath | grep -q "built script OK"\necho GOOD'})

        if os.getuid() == 0:
            # Add Build-Depends-Indep:, Build-Depends-Arch and a build
            # profile package. We can only do this if we're root, because
            # packages that we unpack non-system-wide as non-root are
            # invisible to dpkg-checkbuilddeps so the build would fail.
            subprocess.check_call(['sed', '-i', '-e',
                                   r'/^Build-Depends:/ a\Build-Depends-Indep: dpkg-dev, nonexisting <cross>',
                                   '-e',
                                   r'/^Build-Depends:/ a\Build-Depends-Arch: aspell-doc',
                                   os.path.join(p, 'debian', 'control')])

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # test should succeed
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'pass\s+PASS', out)

        if os.getuid() == 0:
            self.assertIn('Unpacking aspell-doc', out)

        # should build package
        self.assertIn('dh build', err)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)GOOD\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_isolation(self):
        '''isolation restrictions'''

        p = self.build_src('Tests: ic ic2\nDepends:\nRestrictions: isolation-container\n\n'
                           'Tests: im\nDepends:\nRestrictions: isolation-machine\n',
                           {'ic': '#!/bin/sh\necho container ok',
                            'ic2': '#!/bin/sh\necho container2 ok',
                            'im': '#!/bin/sh\necho machine ok'})

        (code, out, err) = self.runtest(['-B', p])

        if self.has_isolation_machine:
            self.assertRegex(out, r'im\s+PASS', out)
            self.assertIn('\nmachine ok\n', out)
        else:
            self.assertRegex(out, r'im\s+SKIP .*machine', out)
            self.assertNotIn('machine ok', out)

        if not self.has_isolation_container:
            self.assertRegex(out, r'ic\s+SKIP .*container', out)
            self.assertRegex(out, r'ic2\s+SKIP .*container', out)
            self.assertNotIn('container ok', out)
            self.assertNotIn('container2 ok', out)
        else:
            self.assertRegex(out, r'ic\s+PASS', out)
            self.assertRegex(out, r'ic2\s+PASS', out)
            self.assertIn('container ok\n', out)
            self.assertIn('container2 ok\n', out)

        if self.has_isolation_machine and self.has_isolation_container:
            # all tests run
            self.assertEqual(code, 0, out + err)
        elif self.has_isolation_machine or self.has_isolation_container:
            # one test run, one skipped
            self.assertEqual(code, 2, out + err)
        else:
            # all skipped
            self.assertEqual(code, 8, out + err)

    def test_breaks_testbed(self):
        '''breaks-testbed restriction'''

        p = self.build_src('Tests: zap boom\nDepends:\nRestrictions: allow-stderr needs-root breaks-testbed',
                           {'zap': '#!/bin/sh\ntouch /zap\n'
                            '[ -d "$HOME" ] || echo "HOME not set" >&2\n'
                            'apt-get install -y aspell-doc\n',
                            'boom': '#!/bin/sh -e\n[ ! -e /zap ]; [ ! -e /usr/share/doc/aspell-doc ]; touch /boom'})

        (code, out, err) = self.runtest(['--no-built-binaries', p, '-d'])

        if not self.can_revert_full_system:
            # test should be skipped as these runners doesn't provide revert-full-system
            self.assertEqual(code, 8, err)
            self.assertRegex(out, r'zap\s+SKIP Test breaks testbed')
            # FIXME: we should see this too!
            # self.assertRegex(out, r'boom\s+SKIP Test breaks testbed')
            self.assertNotIn(out, 'no tests')
        else:
            # both tests should run; the second one (boom) should not see the
            # effect of the first one (/zap existing and installing aspell-doc)
            self.assertEqual(code, 0, err)
            self.assertRegex(out, r'zap\s+PASS')
            self.assertRegex(out, r'boom\s+PASS')
            self.assertIn('Unpacking aspell-doc', out)

    def test_flaky_success(self):
        '''A flaky test succeeds'''
        p = self.build_src('Tests: unreproducible\nRestrictions: flaky\nDepends: coreutils\n',
                           {'unreproducible': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'unreproducible\s+PASS', out)

    def test_skippable_success(self):
        '''A skippable test succeeds'''
        p = self.build_src('Tests: needs-magic\nRestrictions: skippable\nDepends: coreutils\n',
                           {'needs-magic': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'needs-magic\s+PASS', out)

    def test_needs_internet_success(self):
        '''A needs-internet test succeeds'''
        p = self.build_src('Tests: downloads-data\nRestrictions: needs-internet\nDepends: coreutils\n',
                           {'downloads-data': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'downloads-data\s+PASS', out)

    def test_needs_internet_skipped(self):
        '''A needs-internet test is skipped'''
        p = self.build_src('Tests: downloads-data\nRestrictions: needs-internet\nDepends: coreutils\n',
                           {'downloads-data': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', '--needs-internet=skip', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'downloads-data\s+SKIP Test needs unrestricted internet', out)

    def test_needs_internet_tried_success(self):
        '''A needs-internet test is tried and succeeds'''
        p = self.build_src('Tests: downloads-data\nRestrictions: needs-internet\nDepends: coreutils\n',
                           {'downloads-data': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', '--needs-internet=try', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'downloads-data\s+PASS', out)

    def test_needs_internet_tried_skipped(self):
        '''A needs-internet test is tried and skipped'''
        p = self.build_src('Tests: downloads-data\nRestrictions: needs-internet\nDepends: coreutils\n',
                           {'downloads-data': '#!/bin/sh\necho I am sick\nexit 7\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', '--needs-internet=try', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'downloads-data\s+SKIP Failed, but test has needs-internet', out)

    @unittest.skipIf(host_arch != 'amd64', 'needs to run on amd64')
    def test_arch_in_supported_list(self):
        '''A test on an explicitly suppported architecture succeeds'''
        p = self.build_src('Tests: pass\nArchitecture: amd63 amd64\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

    def test_arch_not_in_negated_list(self):
        '''A test on an implicitly supported architecture succeeds'''
        p = self.build_src('Tests: pass\nArchitecture: !amd63 !amd63\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

    @unittest.skipIf(host_arch != 'amd64', 'needs to run on amd64')
    def test_arch_in_unsupported_list(self):
        '''A test on an explicitly unsuppported architecture is skipped'''
        p = self.build_src('Tests: skip-me\nArchitecture: !amd64\nDepends: coreutils\n',
                           {'skip-me': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'skip-me\s+SKIP Test declares architecture as not supported', out)

    @unittest.skipIf(host_arch != 'amd64', 'needs to run on amd64')
    def test_arch_in_unsupported_wildcard_list(self):
        '''A test on an explictly unsuppported wildcard architecture is skipped'''
        p = self.build_src('Tests: skip-me\nArchitecture: !linux-any\nDepends: coreutils\n',
                           {'skip-me': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'skip-me\s+SKIP Test declares architecture as not supported', out)

    def test_arch_not_in_supported_list(self):
        '''A test on an implicitly unsupported architecture is skipped'''
        p = self.build_src('Tests: skip-me\nArchitecture: amd63\nDepends: coreutils\n',
                           {'skip-me': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'skip-me\s+SKIP Test lists explicitly supported architectures', out)

    @unittest.skipIf(host_arch == 'hurd', 'needs to run on !hurd')
    def test_arch_not_in_supported_wildcard_list(self):
        '''A test on an implicitly unsupported wildcard architecture is skipped'''
        p = self.build_src('Tests: skip-me\nArchitecture: hurd-any\nDepends: coreutils\n',
                           {'skip-me': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'skip-me\s+SKIP Test lists explicitly supported architectures', out)

    def test_arch_in_mixed_list_skipped_wildcard(self):
        '''A test in an mixed list on supported architecture succeeds'''
        p = self.build_src('Tests: skip-me\nArchitecture: !amd63 linux-any\nDepends: coreutils\n',
                           {'skip-me': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'skip-me\s+SKIP It is not', out)

    @unittest.skipIf(host_arch != 'amd64', 'needs to run on amd64')
    def test_arch_in_mixed_list_skipped_explicit(self):
        '''A test in an mixed list on unsupported architecture is skipped'''
        p = self.build_src('Tests: skip-me\nArchitecture: !amd64 linux-any\nDepends: coreutils\n',
                           {'skip-me': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'skip-me\s+SKIP Test declares architecture as not supported', out)

    def test_arch_any(self):
        '''A test on "any" architecture succeeds'''
        p = self.build_src('Tests: pass\nArchitecture: any\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

    def test_arch_all(self):
        '''A test on "all" architecture skipped'''
        p = self.build_src('Tests: skip-me\nArchitecture: all\nDepends: coreutils\n',
                           {'skip-me': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r"skip-me\s+SKIP Arch 'all' not", out)


class DebTestsFailureModes:
    '''Common deb tests for handling various failure modes

    These are not sensitive to the virt runner, thus are ony run with fast
    backends (null, chroot) so that the testsuite does not take longer than
    necessary.
    '''
    def test_tree_norestrictions_nobuild_fail_on_exit(self):
        '''source tree, no build, no restrictions, test fails with non-zero'''

        p = self.build_src('Tests: nz\nDepends: coreutils\n',
                           {'nz': '#!/bin/sh\necho I am sick\nexit 7'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # test should fail
        self.assertEqual(code, 4)
        self.assertRegex(out, r'nz\s+FAIL non-zero exit status 7')

        # should show test stdout
        self.assertRegex(out, r'(^|\n)I am sick\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)
        # should not build package
        self.assertNotIn('dh build', err)
        # should show summary
        self.assertRegex(err, r'@@@ summary\nnz\s+FAIL non-zero exit status 7\n$')

    def test_tree_norestrictions_nobuild_fail_on_stderr(self):
        '''source tree, no build, no restrictions, test fails with stderr'''

        p = self.build_src('Tests: se\nDepends: coreutils\n',
                           {'se': '#!/bin/sh\necho I am sick >&2\n'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # test should fail
        self.assertEqual(code, 4)
        self.assertRegex(out, r'(^|\n)se\s+FAIL stderr: I am sick\n')

        # should show test stderr
        self.assertRegex(err, 'stderr [ -]+\nI am sick', err)

    def test_tree_allow_stderr_nobuild_fail_on_exit(self):
        '''source tree, no build, allow-stderr, test fails with non-zero'''

        p = self.build_src('Tests: nz\nDepends: coreutils\nRestrictions: allow-stderr',
                           {'nz': '#!/bin/sh\necho I am fine >&2\necho babble\nexit 7'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # test should fail
        self.assertEqual(code, 4)
        self.assertRegex(out, r'nz\s+FAIL non-zero exit status 7')

        # should show test stdout/err inline
        self.assertRegex(out, r'(^|\n)babble\n')
        self.assertRegex(err, '---+\nI am fine\nautopkgtest', err)
        # but not complain about stderr
        self.assertNotIn(' stderr ', err)

    def test_tree_norestrictions_nobuild_fail_on_stderr_and_exit(self):
        '''source tree, no build, no restrictions, test fails with stderr+exit'''

        p = self.build_src('Tests: senz\nDepends:\n',
                           {'senz': '#!/bin/sh\necho I am sick >&2\nexit 7\n'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # test should fail
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'senz\s+FAIL non-zero exit status 7')

        # should show test stderr, but inline
        self.assertRegex(err, 'stderr [ -]+\nI am sick', err)

    def test_tree_allow_stderr_nobuild_success(self):
        '''source tree, no build, allow-stderr, test success'''

        p = self.build_src('Tests: pass\nDepends: coreutils\nRestrictions: allow-stderr',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout/err
        self.assertIn('babble\n', out)
        self.assertRegex(err, r'-+\nI am fine\nautopkgtest \[[0-9: -]+\]: test pass: --', err)
        # but not complain about stderr
        self.assertNotIn(' stderr ', err)

        # should log package version
        self.assertIn('testing package testpkg version 1\n', err)

        # no restricted dependencies functionality as they are already installed
        self.assertNotIn('will only work for some packages', err)

    def test_flaky_fail(self):
        '''A flaky test fails'''
        p = self.build_src('Tests: unreproducible\nRestrictions: flaky\nDepends: coreutils\n',
                           {'unreproducible': '#!/bin/sh\nexit 1\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'unreproducible\s+FLAKY', out)

    def test_flaky_fail_mixed(self):
        '''A flaky test fails, but another test passes'''
        p = self.build_src('Tests: unreproducible ok\nRestrictions: flaky\nDepends: coreutils\n',
                           {'unreproducible': '#!/bin/sh\nexit 1\n',
                            'ok': '#!/bin/sh\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, r'unreproducible\s+FLAKY', out)
        self.assertRegex(out, r'ok\s+PASS', out)

    def test_skippable_skipped(self):
        '''A skippable test skips itself'''
        p = self.build_src('Tests: needs-magic\nRestrictions: skippable\nDepends: coreutils\n',
                           {'needs-magic': '#!/bin/sh\necho Cannot run >&2\nexit 77\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'needs-magic\s+SKIP', out)

    def test_skippable_skipped_mixed(self):
        '''A skippable test skips itself, but another test passes'''
        p = self.build_src('Tests: needs-magic ok\nRestrictions: skippable\nDepends: coreutils\n',
                           {'needs-magic': '#!/bin/sh\necho Cannot run >&2\nexit 77\n',
                            'ok': '#!/bin/sh\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, r'needs-magic\s+SKIP', out)
        self.assertRegex(out, r'ok\s+PASS', out)

    def test_skippable_fail(self):
        '''A skippable test fails'''
        p = self.build_src('Tests: needs-magic\nRestrictions: skippable\nDepends: coreutils\n',
                           {'needs-magic': '#!/bin/sh\nexit 1\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'needs-magic\s+FAIL non-zero exit status 1', out)

    def test_unskippable(self):
        '''A non-skippable test fails with an exit status that happens
        to match the skippable API.'''
        p = self.build_src('Tests: no-magic\nDepends: coreutils\n',
                           {'no-magic': '#!/bin/sh\nexit 77\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'no-magic\s+FAIL non-zero exit status 77', out)

    def test_superficial_success(self):
        '''If all successful tests are superficial, it's as though they
        were skipped.'''
        p = self.build_src('Tests: toy\nRestrictions: superficial\nDepends: coreutils\n',
                           {'toy': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        # No non-superficial tests passed
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'toy\s+PASS\s+\(superficial\)', out)

    def test_superficial_success_mixed(self):
        '''Superficial tests don't harm non-superficial tests.'''
        p = self.build_src('Tests: toy\nRestrictions: superficial\nDepends: coreutils\n'
                           '\n'
                           'Tests: real\nDepends: coreutils\n',
                           {'toy': '#!/bin/sh\necho I am fine\n',
                            'real': '#!/bin/sh\necho I am also fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'toy\s+PASS\s+\(superficial\)', out)
        self.assertRegex(out, r'real\s+PASS', out)

    def test_superficial_fail(self):
        '''A superficial test failing is the same as if it was
        non-superficial'''
        p = self.build_src('Tests: toy\nRestrictions: superficial\nDepends: coreutils\n',
                           {'toy': '#!/bin/sh\nexit 1\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'toy\s+FAIL non-zero exit status 1', out)

    def test_superficial_fail_mixed(self):
        '''A superficial test (among others) failing is the same as if it
        was non-superficial'''
        p = self.build_src('Tests: toy\nRestrictions: superficial\nDepends: coreutils\n'
                           '\n'
                           'Tests: real\nDepends: coreutils\n',
                           {'toy': '#!/bin/sh\nexit 1\n',
                            'real': '#!/bin/sh\necho I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'toy\s+FAIL non-zero exit status 1', out)
        self.assertRegex(out, r'real\s+PASS', out)

    def test_unknown_restriction(self):
        '''Tests with unknown restrictions are skipped

        Each test is reported separately to avoid confusing output
        (#851232)'''
        p = self.build_src('Tests: foo bar\nRestrictions: needs-more-cowbell\nDepends: coreutils\n',
                           {'foo': '#!/bin/sh\nexit 1\n',
                            'bar': '#!/bin/sh\nexit 1\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'foo\s+SKIP unknown restriction', out)
        self.assertRegex(out, r'bar\s+SKIP unknown restriction', out)


class NullRunner(AdtTestCase, DebTestsAll, DebTestsFailureModes):
    def __init__(self, *args, **kwargs):
        super(NullRunner, self).__init__(['null'], *args, **kwargs)

    def test_tree_output_dir(self):
        '''source tree, explicit --output-dir

        This also covers using upper-case lettes in test names.
        '''
        p = self.build_src('Tests: sP sF\nDepends: coreutils\n\n'
                           'Tests: bP\nDepends: coreutils\nRestrictions: build-needed',
                           {'sP': '#!/bin/sh\n./test_static\n',
                            'sF': '#!/bin/sh\necho kaputt >&2',
                            'bP': '#!/bin/sh\n./test_built'})

        # change to funny version
        subprocess.check_call(['sed', '-i', 's/(1)/(1.2+git3.4-0foo1~testy)/',
                               os.path.join(p, 'debian', 'changelog')])

        outdir = os.path.join(self.workdir, 'out')
        # check pre-existing, but empty dir
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['--no-built-binaries', p,
                                         '--env=COLOR=blue',
                                         '--env=SOMETHING=foo bar',
                                         '--output-dir=' + outdir])

        # test results
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'sP\s+PASS', out)
        self.assertRegex(out, r'sF\s+FAIL stderr: kaputt', out)
        self.assertRegex(out, r'bP\s+PASS', out)

        # should show test stdout and stderr
        self.assertRegex(out, r'(^|\n)static script OK\n')
        self.assertIn('\nbuilt script OK\n', out)
        self.assertRegex(err, r'stderr [ -]+\nkaputt', err)

        # should show summary at the end
        self.assertRegex(err, r'@@@ summary\nsP\s+PASS\nsF\s+FAIL stderr: kaputt\nbP\s+PASS\n$')

        # should build package
        self.assertIn('dh build', err)

        # check outdir test stdout/err
        with open(os.path.join(outdir, 'sP-stdout')) as f:
            self.assertEqual(f.read(), 'static script OK\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'sP-stderr')))
        with open(os.path.join(outdir, 'bP-stdout')) as f:
            self.assertEqual(f.read(), 'built script OK\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'bP-stderr')))
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'sF-stdout')))
        with open(os.path.join(outdir, 'sF-stderr')) as f:
            self.assertEqual(f.read(), 'kaputt\n')

        # check outdir log
        with open(os.path.join(outdir, 'log')) as f:
            contents = f.read()
        self.assertIn('build needed for tests', contents)
        self.assertIn('dh build', contents)
        self.assertRegex(contents, r'sF\s+FAIL stderr: kaputt')
        self.assertIn('testing package testpkg version 1.2+git3.4-0foo1~testy\n', contents)
        self.assertRegex(err, r'@@@ summary\nsP\s+PASS\nsF\s+FAIL stderr: kaputt\nbP\s+PASS\n$')

        # check summary
        with open(os.path.join(outdir, 'summary')) as f:
            self.assertRegex(f.read(), r'^sP\s+PASS\nsF\s+FAIL stderr: kaputt\nbP\s+PASS$')

        # check test package version
        self.assertIn('testing package testpkg version 1.2+git3.4-0foo1~testy\n', err)
        with open(os.path.join(outdir, 'testpkg-version')) as f:
            contents = f.read()
        self.assertEqual(contents, 'testpkg 1.2+git3.4-0foo1~testy\n')

        # check recorded package lists
        with open(os.path.join(outdir, 'testbed-packages')) as f:
            contents = f.read()
            lines = contents.splitlines()
            self.assertGreater(len(lines), 10)
            self.assertRegex(lines[0], r'^[0-9a-z.-]+\t[0-9a-z.~-]+')
            self.assertRegex(lines[1], r'^[0-9a-z.-]+\t[0-9a-z.~-]+')
            self.assertIn('bash\t', contents)

        # check testinfo
        with open(os.path.join(outdir, 'testinfo.json')) as f:
            info = json.load(f)
        u = os.uname()
        self.assertEqual(info['kernel_version'],
                         '%s %s %s' % (u.sysname, u.release, u.version.strip()))
        self.assertNotIn('test_kernel_versions', info)
        self.assertEqual(info['custom_environment'],
                         ['COLOR=blue', 'SOMETHING=foo bar'])
        self.assertEqual(info['virt_server'], 'autopkgtest-virt-null')
        self.assertEqual(info['nproc'],
                         subprocess.check_output(['nproc'], universal_newlines=True).strip())
        if os.uname()[4] in ['x86_64', 'i686']:
            self.assertIn('cpu_model', info)
            self.assertIn('cpu_flags', info)

        # test don't pull in any additional dependencies
        for t in ['sP', 'sF', 'bP']:
            self.assertEqual(os.path.getsize(os.path.join(
                outdir, '%s-packages' % t)), 0)

        # check for cruft in outdir
        # --no-built-binaries, we don't expect any debs
        files = [i for i in os.listdir(outdir)
                 if not fnmatch.fnmatch(i, '[sb]*-std*') and
                 not fnmatch.fnmatch(i, '[sb]*-packages')]
        self.assertEqual(set(files), set(['log', 'summary', 'testpkg-version',
                                          'testbed-packages', 'testinfo.json']))

    def test_tree_output_dir_nonempty(self):
        '''existing and non-empty --output-dir'''

        p = self.build_src('Tests: p\nDepends:\n', {'p': '#!/bin/sh\ntrue\n'})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)
        # put some cruft into it
        with open(os.path.join(outdir, 'cruft.txt'), 'w') as f:
            f.write('hello world')

        (code, out, err) = self.runtest(['--no-built-binaries', p,
                                         '--output-dir=' + outdir])

        # test results
        self.assertEqual(code, 20, err)
        self.assertEqual(out, '')
        self.assertRegex(err, r'--output-dir.*not empty')

        # file is still there
        with open(os.path.join(outdir, 'cruft.txt')) as f:
            self.assertEqual(f.read(), 'hello world')
        # nothing else got written there
        self.assertEqual(os.listdir(outdir), ['cruft.txt'])

    def test_tree_output_dir_in_test_tree(self):
        '''source tree, --output-dir in tests tree

        NB that this is a pathological case, but easy to run into.
        '''
        p = self.build_src('Tests: s\nDepends:\n',
                           {'s': '#!/bin/sh\necho OK'})

        outdir = os.path.join(p, 'out')

        (code, out, err) = self.runtest(['-B', p, '--output-dir=' + outdir])

        # test results
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r's\s+PASS', out)

        # should show test stdout and stderr
        self.assertRegex(out, r'(^|\n)OK\n')

        # check outdir test stdout/err
        with open(os.path.join(outdir, 's-stdout')) as f:
            self.assertEqual(f.read(), 'OK\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 's-stderr')))

        # check outdir log
        with open(os.path.join(outdir, 'log')) as f:
            contents = f.read()
        self.assertRegex(contents, r's\s+PASS')

    def test_tree_apply_patches_build_needed(self):
        '''source tree, 3.0 (quilt) patches get applied with build-needed'''

        p = self.build_src('Tests: pass\nDepends: coreutils\nRestrictions: build-needed\n',
                           {'pass': '#!/bin/sh -e\ndebian/testpkg/usr/bin/test_built\n'
                            'debian/testpkg/usr/bin/test_static; ./test_static\n'})

        # add patch
        patchdir = os.path.join(p, 'debian', 'patches')
        os.mkdir(patchdir)
        with open(os.path.join(patchdir, '01_hack.patch'), 'w') as f:
            f.write('''--- testpkg.orig/test_static
+++ testpkg/test_static
@@ -1,2 +1,2 @@
 #!/bin/sh
-echo "static script OK"
+echo "static patched script OK"
''')
        with open(os.path.join(patchdir, 'series'), 'w') as f:
            f.write('01_hack.patch')

        # turn into 3.0 (quilt) source
        dsrcdir = os.path.join(p, 'debian', 'source')
        os.mkdir(dsrcdir)
        with open(os.path.join(dsrcdir, 'format'), 'w') as f:
            f.write('3.0 (quilt)\n')

        # run tests, should apply unapplied patches
        (code, out, err) = self.runtest(['-B', '-d', p])

        # test should succeed
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should have patched source
        self.assertRegex(err, r'dpkg-source:.*01_hack.patch')
        self.assertIn('built script OK\nstatic patched script OK\nstatic patched script OK\n', out)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should build package
        self.assertIn('dh build', err)

    def test_tree_apply_patches_nobuild(self):
        '''source tree, 3.0 (quilt) patches get applied with built package'''

        p = self.build_src('Tests: pass\nDepends: coreutils',
                           {'pass': '#!/bin/sh -e\n./test_static\n'})

        # add patch
        patchdir = os.path.join(p, 'debian', 'patches')
        os.mkdir(patchdir)
        with open(os.path.join(patchdir, '01_hack.patch'), 'w') as f:
            f.write('''--- testpkg.orig/test_static
+++ testpkg/test_static
@@ -1,2 +1,2 @@
 #!/bin/sh
-echo "static script OK"
+echo "static patched script OK"
''')
        with open(os.path.join(patchdir, 'series'), 'w') as f:
            f.write('01_hack.patch')

        # turn into 3.0 (quilt) source
        dsrcdir = os.path.join(p, 'debian', 'source')
        os.mkdir(dsrcdir)
        with open(os.path.join(dsrcdir, 'format'), 'w') as f:
            f.write('3.0 (quilt)\n')

        # run tests, should apply unapplied patches
        (code, out, err) = self.runtest(['-B', '-d', p])

        # test should succeed
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should have patched source
        self.assertRegex(err, r'dpkg-source:.*01_hack.patch')
        self.assertIn('static patched script OK\n', out)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not build package
        self.assertNotIn('dh build', err)

    def test_dsc_norestrictions_nobuild_success(self):
        '''dsc, no build, no restrictions, test success'''

        p = self.build_dsc('Tests: pass\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertIn('coreutils', err)
        self.assertRegex(out, r'pass\s+PASS', out)

        self.assertIn('processing dependency coreutils', err)
        # should show test stdout
        self.assertRegex(out, r'(^|\n)I am fine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should log package version
        self.assertIn('testing package testpkg version 1\n', err)

    def test_dev_stdouterr_access(self):
        '''write to /dev/stdout and /dev/stderr in a test'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr',
                           {'pass': '#!/bin/sh\necho I am fine >/dev/stdout\n'
                            'echo SomeDebug >/dev/stderr\n'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout/err
        self.assertIn('I am fine\n', out)
        # should show test stderr
        self.assertIn('\nSomeDebug\n', err)

    def test_summary(self):
        '''--summary option'''

        p = self.build_src('Tests: good bad\nDepends:\n',
                           {'good': '#!/bin/sh\necho happy\n',
                            'bad': '#!/bin/sh\nexit 1'})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--no-built-binaries', p,
                                         '--summary=' + summary])

        # test results
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'good\s+PASS', out)
        self.assertRegex(out, r'bad\s+FAIL non-zero exit status 1', out)

        # check summary file
        with open(summary) as f:
            self.assertEqual(f.read(), '''good                 PASS
bad                  FAIL non-zero exit status 1
''')

    def test_timeout(self):
        '''handling test timeout'''

        p = self.build_dsc('Tests: totest\nDepends:\n',
                           {'totest': '#!/bin/sh\necho start\nsleep 10\necho after_sleep\n'})

        (code, out, err) = self.runtest(['--no-built-binaries', '--timeout-test=3', p])
        # test should time out
        self.assertEqual(code, 4, err)
        self.assertIn("timed out on command", err)
        self.assertRegex(out, r'totest\s+FAIL timed out', out)

        # should show test stdout
        self.assertIn('start', out)
        # but not let the test finish
        self.assertNotIn('after_sleep', out)

    def test_timeout_no_output(self):
        '''handling test timeout for test without any output'''

        # use pid-specific name here to avoid clashes during multiple parallel runs
        procname = 'mysleep%i' % os.getpid()

        p = self.build_dsc('Tests: totest\nDepends:\n',
                           {'totest': '#!/bin/bash\nexec -a %s sleep 15\n' % procname})

        time_start = time.time()
        (code, out, err) = self.runtest(['--no-built-binaries', '--timeout-test=3', p])
        duration = time.time() - time_start

        # test should time out
        self.assertEqual(code, 4, err)
        self.assertIn("timed out on command", err)
        self.assertRegex(out, r'totest\s+FAIL timed out', out)
        self.assertLess(duration, 10, err)

        # should not leave test process running
        self.assertEqual(subprocess.getoutput('pidof ' + procname), '')

    def test_timeout_long_test(self):
        '''long-running test with custom timeouts

        This verifies that the right timeout is being used for tests.
        '''
        p = self.build_dsc('Tests: p\nDepends:\nRestrictions: build-needed',
                           {'p': '#!/bin/sh\necho start\nsleep 5\n./test_built\n'})

        (code, out, err) = self.runtest(['-B', '--timeout-test=6',
                                         '--timeout-build=20', p])
        # test should not time out
        self.assertEqual(code, 0, err)
        self.assertNotIn('timed out', err)

        # should show test stdout
        self.assertIn('start\n', out)
        self.assertIn('built script OK\n', out)

    def test_timeout_long_build(self):
        '''long-running build with custom timeouts

        This verifies that the right timeout is being used for builds.
        '''
        p = self.build_src('Tests: p\nDepends:\nRestrictions: build-needed',
                           {'p': '#!/bin/sh\necho start\n\n./test_built\n'})

        # make build take 4s
        subprocess.check_call(['sed', '-i', '/^build:/ s/$/\\n\\tsleep 4/',
                               os.path.join(p, 'Makefile')])

        (code, out, err) = self.runtest(['--timeout-test=1', '--timeout-build=30', '-B', p])
        # should build package
        self.assertIn('dh build', err)

        # test should not time out
        self.assertEqual(code, 0, err)
        self.assertNotIn('timed out', err)

        # should show test stdout
        self.assertIn('start\n', out)
        self.assertIn('built script OK\n', out)

    def test_timeout_long_build_fail(self):
        '''long-running build times out

        This verifies that the right timeout is being used for builds.
        '''
        p = self.build_src('Tests: p\nDepends:\nRestrictions: build-needed',
                           {'p': '#!/bin/sh\necho start\n\n./test_built\n'})

        # make build take 4s
        subprocess.check_call(['sed', '-i', '/^build:/ s/$/\\n\\tsleep 4/',
                               os.path.join(p, 'Makefile')])

        (code, out, err) = self.runtest(['--timeout-test=1', '--timeout-build=6', '-B', p])
        # should start building package
        self.assertIn('dh build', err)

        # build should time out
        self.assertEqual(code, 16, err)
        self.assertIn('timed out', err)

        # should not start tests
        self.assertNotIn('start\n', out)

    def test_logfile_success(self):
        '''--log-file option, success'''

        p = self.build_src('Tests: pass\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        logfile = os.path.join(self.workdir, 'test.log')
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p,
                                         '--log-file=' + logfile])
        # test should succeed
        self.assertEqual(code, 0, err)

        with open(logfile) as f:
            log = f.read()
        self.assertIn('coreutils', err)
        self.assertIn('coreutils', log)
        self.assertRegex(out, r'pass\s+PASS')
        self.assertRegex(log, r'pass\s+PASS')

        self.assertIn('processing dependency coreutils', err)
        self.assertIn('processing dependency coreutils', log)
        # should show test stdout
        self.assertRegex(out, r'(^|\n)I am fine\n')
        self.assertRegex(log, r'(^|\n)I am fine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)
        self.assertNotIn(' stderr ', log)

        # should not build package
        self.assertNotIn('dh build', err)

    def test_logfile_failure(self):
        '''--log-file option, failure'''

        p = self.build_src('Tests: nz\nDepends: coreutils\nRestrictions: build-needed\n',
                           {'nz': '#!/bin/sh\n./test_built\necho I am sick >&2\nexit 7'})

        logfile = os.path.join(self.workdir, 'test.log')
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p, '--log-file=' + logfile])
        with open(logfile) as f:
            log = f.read()

        # test should fail
        self.assertEqual(code, 4)
        self.assertRegex(out, r'nz\s+FAIL non-zero exit status 7')
        self.assertRegex(log, r'nz\s+FAIL non-zero exit status 7')

        self.assertIn('processing dependency coreutils', err)
        self.assertIn('processing dependency coreutils', log)

        # should build package
        self.assertIn('dh build', err)
        self.assertIn('dh build', log)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)built script OK\n')
        self.assertRegex(log, r'(^|\n)built script OK\n')
        # should show test stderr
        self.assertRegex(err, r'stderr [ -]+\nI am sick\n')
        self.assertRegex(log, r'stderr [ -]+\nI am sick\n')

    def test_unicode(self):
        '''Unicode test output'''

        p = self.build_src('Tests: se\nDepends:\n',
                           {'se': '#!/bin/sh\necho ‘a♩’; echo fancy ‴ʎɔuɐɟ″!>&2'})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--no-built-binaries', p, '--summary=' + summary])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'se\s+FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

        # should show test stdout/stderr
        self.assertIn('‘a♩’\n', out)
        self.assertRegex(err, r'stderr [ -]+\nfancy ‴ʎɔuɐɟ″!\n')

        with open(summary, encoding='UTF-8') as f:
            self.assertEqual(f.read(), 'se                   FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

    def test_non_unicode_stdout(self):
        '''Don't fail on non-unicode test output'''

        p = self.build_src('Tests: se\nDepends:\n',
                           {'se': '#!/bin/sh\necho aß | iconv -f UTF-8 -t ISO-8859-1; echo fancy ‴ʎɔuɐɟ″!>&2'})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--no-built-binaries', p, '--summary=' + summary])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'se\s+FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

        # should show test stdout/stderr
        self.assertIn('a�\n', out)
        self.assertRegex(err, r'stderr [ -]+\nfancy ‴ʎɔuɐɟ″!\n')

        with open(summary, encoding='UTF-8') as f:
            self.assertEqual(f.read(), 'se                   FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

    def test_non_unicode_stderr(self):
        '''Don't fail on non-unicode test output on stderr'''

        p = self.build_src('Tests: se\nDepends:\n',
                           {'se': '#!/bin/sh\necho ‘a♩’; echo bß | iconv -f UTF-8 -t ISO-8859-1 >&2'})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--no-built-binaries', p, '--summary=' + summary])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'se\s+FAIL stderr: b�\n')

        # should show test stdout/stderr
        self.assertIn('‘a♩’\n', out)
        self.assertRegex(err, r'stderr [ -]+\nb�')

        with open(summary, encoding='UTF-8') as f:
            self.assertEqual(f.read(), 'se                   FAIL stderr: b�\n')

    def test_test_command_with_single_quotes(self):
        '''Test-Command contains double quotes inside single quotes'''

        p = self.build_src("Test-Command: echo 'bla'\nDepends:\n",
                           {})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'\ncommand1             PASS\n')

    def test_test_command_with_double_quotes_in_single_quotes(self):
        '''Test-Command contains double quotes inside single quotes'''

        p = self.build_src("Test-Command: python -c 'import anosql; print(\"Anosql successfully installed\")'\nDepends:\n",
                           {})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'command1             FAIL non-zero exit status')

    def test_test_command_with_double_quotes_and_single_quotes(self):
        '''Test-Command contains quotes'''

        p = self.build_src("Test-Command: echo \"x y z\" | awk '{print $1$3}' >&2\nDepends:\n",
                           {})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'command1             FAIL stderr: xz\n')

    def test_no_tests_dir(self):
        '''package without debian/tests/'''

        p = self.build_src(None, None)
        (code, out, err) = self.runtest([p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'SKIP no tests in this package', out)

    @unittest.skipIf(os.getuid() == 0, 'needs to run as user')
    def test_no_apt_install_with_missing_dependency(self):
        '''package with missing tests dependencies without available apt-get install'''
        p = self.build_src('Test-Command: /bin/true\nDepends: package-that-does-not-exist\n', {})
        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 12, err)

    @unittest.skipIf(os.getuid() == 0, 'needs to run as user')
    def test_no_apt_install_with_dependencies_satisfied(self):
        '''package with dependencies satisfied while apt-get install is not available'''
        p = self.build_src('Test-Command: /bin/true\nDepends: coreutils\n', {})
        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 0, err)

    def test_test_command(self):
        '''Test-Command: instead of Tests:'''

        p = self.build_src('Test-Command: echo "Some Stdout"\nDepends:\n\n'
                           'Test-Command: echo "Some StdErr" >&2; sleep 0.5; echo done\nDepends:\n'
                           'Restrictions: allow-stderr\n\n'
                           'Test-Command: echo "More Stderr" >&2; echo hello > $AUTOPKGTEST_ARTIFACTS/world.txt\nDepends:\n',
                           {})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['-B', p, '-o', outdir, '-d'])
        # two pass, one fails
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'done\n')
        self.assertRegex(out, r'command1\s+PASS')
        self.assertRegex(out, r'command2\s+PASS')
        self.assertRegex(out, r'command3\s+FAIL stderr: More Stderr')
        self.assertRegex(err, r'stderr [ -]+\nMore Stderr\n')
        # shows commands
        self.assertIn('test command1: echo "Some Stdout"\n', err)
        self.assertIn('test command2: echo "Some StdErr" >&2; sleep 0.5; echo done\n', err)
        self.assertIn('test command3: echo "More Stderr" >&2; echo hello > $AUTOPKGTEST_ARTIFACTS/world.txt\n', err)

        # check artifacts
        with open(os.path.join(outdir, 'command1-stdout')) as f:
            self.assertEqual(f.read(), 'Some Stdout\n')
        with open(os.path.join(outdir, 'command2-stdout')) as f:
            self.assertEqual(f.read(), 'done\n')
        with open(os.path.join(outdir, 'command2-stderr')) as f:
            self.assertEqual(f.read(), 'Some StdErr\n')
        with open(os.path.join(outdir, 'command3-stderr')) as f:
            self.assertEqual(f.read(), 'More Stderr\n')
        with open(os.path.join(outdir, 'artifacts', 'world.txt')) as f:
            self.assertEqual(f.read(), 'hello\n')

    @unittest.skipUnless(have_apt_src, 'this test needs local deb-src apt sources')
    def test_apt_source_nonexisting(self):
        '''apt-source for nonexisting package'''

        (code, out, err) = self.runtest(['no.such-package'])

        self.assertEqual(code, 12, err)
        self.assertRegex(err, r'no.such-package')
        self.assertNotIn('PASS', out)
        self.assertNotIn('[----', out)

    def test_undeletable_files(self):
        '''source tree has undeletable files'''

        p = self.build_src('Tests: t1 t2\nDepends:\n',
                           {'t1': '#!/bin/sh\necho eins\n',
                            't2': '#!/bin/sh\necho zwei\n'})
        os.mkdir(os.path.join(p, 'data'))
        with open(os.path.join(p, 'data', 'data.txt'), 'w') as f:
            f.write('data\n')
        os.chmod(os.path.join(p, 'data'), 0o555)

        (code, out, err) = self.runtest(['-B', p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r't1\s+PASS', out)
        self.assertRegex(out, r't2\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'^eins\n')
        self.assertRegex(out, r'\nzwei\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # chown it back so that we can clean up
        os.chmod(os.path.join(p, 'data'), 0o755)

    def test_signal(self):
        '''tests keep default signal handler'''

        p = self.build_src('Tests: int\nDepends:',
                           {'int': '''#!/usr/bin/perl -wl
$| = 1; # unbuffer output
my $pid = fork;
if ($pid) { # parent
    sleep 1;
    print "P: killing";
    kill INT => $pid;
    print "P: waiting";
    wait;
    print "P: done";
} else { # child
    sleep;
    print "C: survived!";
}
'''})

        (code, out, err) = self.runtest(['-B', '--timeout-test=20', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'int\s+PASS')

        self.assertIn('P: done\n', out)
        self.assertNotIn('survived', out)

    def test_tree_debian_tests_only(self):
        '''tree with only debian/tests/'''

        srcdir = os.path.join(self.workdir, 'testpkg')
        testdir = os.path.join(srcdir, 'debian', 'tests')
        os.makedirs(testdir)
        with open(os.path.join(testdir, 'control'), 'w') as f:
            f.write('Test-Command: echo IamFine\nDepends:\n')

        (code, out, err) = self.runtest(['-d', '-B', srcdir])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'command1\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)IamFine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_testname(self):
        '''Run only one specified test'''

        p = self.build_src('Tests: one two three\nDepends:',
                           {'one': '#!/bin/sh\necho 1_ONE',
                            'two': '#!/bin/sh\necho 2_TWO',
                            'three': '#!/bin/sh\necho 3_THREE'})

        (code, out, err) = self.runtest(['-B', '--test-name', 'two', p])

        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'two\s+PASS', out)
        self.assertNotIn('one', out)
        self.assertNotIn('three', out)

        self.assertIn('2_TWO', out)
        self.assertNotIn('1_ONE', out)
        self.assertNotIn('3_THREE', out)

    def test_testname_with_others_skipped(self):
        '''Run only one specified test between skipped tests'''

        p = self.build_src('Tests: one three\nDepends:\nRestrictions: needs-quantum-computer\n\nTests: two\nDepends:',
                           {'one': '#!/bin/sh\necho 1_ONE',
                            'two': '#!/bin/sh\necho 2_TWO',
                            'three': '#!/bin/sh\necho 3_THREE'})

        (code, out, err) = self.runtest(['-B', '--test-name', 'two', p])

        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'two\s+PASS', out)
        self.assertNotIn('one', out)
        self.assertNotIn('three', out)

        self.assertIn('2_TWO', out)
        self.assertNotIn('1_ONE', out)
        self.assertNotIn('3_THREE', out)

    def test_testname_noexist(self):
        '''Run only one specified test which does not exist'''

        p = self.build_src('Tests: one two three\nDepends:',
                           {'one': '#!/bin/sh\necho 1_ONE',
                            'two': '#!/bin/sh\necho 2_TWO',
                            'three': '#!/bin/sh\necho 3_THREE'})

        (code, out, err) = self.runtest(['-B', '--test-name', 'four', p])

        self.assertEqual(code, 8, err)
        for w in ['one', 'two', 'three']:
            self.assertNotIn(w, out)

    def test_command_not_found(self):
        '''command-not-found failure is test failure'''

        p = self.build_src('Tests: f\nDepends:\n', {'f': '#!/bin/sh -e\nno_such_command'})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'f\s+FAIL non-zero exit status 127', out)
        # this isn't a failure of the auxverb
        self.assertNotIn('testbed auxverb failed with', err)

    def test_test_not_found(self):
        '''Tests: specifies a nonexisting test'''

        p = self.build_src('Tests: foo\nDepends:\n', {'xxbar': '#!/bin/sh -e\nno_such_command'})

        (code, out, err) = self.runtest(['-B', p])
        self.assertEqual(code, 12, err)
        self.assertRegex(out, r'blame: .*/testpkg', out)
        self.assertIn('badpkg: debian/tests/foo does not exist', out)
        self.assertIn('debian/tests/foo does not exist', err)
        self.assertNotIn('testbed auxverb failed with', err)
        self.assertNotIn('xxbar', out)
        self.assertNotIn('xxbar', err)

    def test_invalid_depends(self):
        '''Depends: has invalid syntax'''

        p = self.build_src('Tests: foo\nDepends: foo bar baz\n', {'foo': '#!/bin/sh -e\nfalse'})

        outdir = os.path.join(p, 'out')
        (code, out, err) = self.runtest(['-B', p, '--output-dir=' + outdir])
        self.assertEqual(code, 12, err)
        self.assertRegex(out, r'blame: .*/testpkg', out)
        self.assertRegex(out, r'badpkg:.*test foo.*Depends field contains an invalid dependency.*foo bar baz')
        self.assertRegex(err, r'erroneous package:.*test foo.*Depends field contains an invalid dependency.*foo bar baz')

        with open(os.path.join(outdir, 'testinfo.json')) as f:
            info = json.load(f)
        self.assertIn(os.uname().sysname, info['kernel_version'])
        self.assertEqual(info['virt_server'], 'autopkgtest-virt-null')

    def test_versioned_depends(self):
        '''Depends: with versions'''

        # ensure that build profile sedding does not affect versioned dependencies
        p = self.build_src('Test-Command: true\nDepends: base-files, coreutils (<< 99:1), bash (>= 0.1), coreutils (>=0.1~)\n', {})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 0, err)
        self.assertIn('install-deps: architecture resolved: base-files, coreutils (<< 99:1), bash (>= 0.1), coreutils (>= 0.1~)', err)

    def test_env_passing(self):
        '''Pass environment variable to test'''

        p = self.build_src('Tests: pass\nDepends: coreutils\n',
                           {'pass': '#!/bin/sh -eu\necho Hey ${NAME}! favourite color: ${FAV_COLOR}.\n'})

        (code, out, err) = self.runtest(
            ['-d', '--env=FAV_COLOR=blue', '--env', 'NAME=Joe', '-B', p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        self.assertIn('processing dependency coreutils', err)
        # should show test stdout
        self.assertRegex(out, r'(^|\n)Hey Joe! favourite color: blue.\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    @unittest.skipUnless(have_git, 'git not installed')
    def test_git_source(self):
        '''Check out source from git (default branch)'''

        p = self.build_src('Tests: pass\nDepends:\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        subprocess.check_call(['git', 'config', '--global', 'user.email', 'autopkgtest@example.com'])
        subprocess.check_call(['git', 'config', '--global', 'user.name', 'Autopkgtest'])
        subprocess.check_call(['git', 'init', '--quiet'], cwd=p)
        subprocess.check_call(['git', 'add', '.'], cwd=p)
        subprocess.check_call(['git', 'commit', '--quiet', '-mfirst'], cwd=p)

        (code, out, err) = self.runtest(['-B', 'file://' + p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)I am fine\n')
        # should show summary
        self.assertRegex(err, r'@@@ summary\npass\s+PASS')

    @unittest.skipUnless(have_git, 'git not installed')
    def test_git_source_branch(self):
        '''Check out source from git (specified branch)'''

        p = self.build_src('Tests: pass\nDepends:\n',
                           {'pass': '#!/bin/sh -e\nls -l branch.txt; echo I am fine\n'})
        subprocess.check_call(['git', 'config', '--global', 'user.email', 'autopkgtest@example.com'])
        subprocess.check_call(['git', 'config', '--global', 'user.name', 'Autopkgtest'])
        subprocess.check_call(['git', 'init', '--quiet'], cwd=p)
        subprocess.check_call(['git', 'add', '.'], cwd=p)
        subprocess.check_call(['git', 'commit', '--quiet', '-mfirst'], cwd=p)
        subprocess.check_call(['git', 'branch', 'wip'], cwd=p)
        subprocess.check_call(['git', 'checkout', '--quiet', 'wip'], cwd=p)
        with open(os.path.join(p, 'branch.txt'), 'w') as f:
            f.write('hello')
        subprocess.check_call(['git', 'add', 'branch.txt'], cwd=p)
        subprocess.check_call(['git', 'commit', '--quiet', '-mdobranch'], cwd=p)
        subprocess.check_call(['git', 'checkout', '--quiet', 'master'], cwd=p)

        (code, out, err) = self.runtest(['-B', 'file://%s#wip' % p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)I am fine\n')
        # should show summary
        self.assertRegex(err, r'@@@ summary\npass\s+PASS')

    def test_tmpdir_env(self):
        '''Respects $TMPDIR'''

        # verify that obsolete $ADTTMP is also set for backwards compatibility
        p = self.build_src('Tests: p\nDepends:\n',
                           {'p': '#!/bin/sh -e\n[ "$ADTTMP" = "$AUTOPKGTEST_TMP" ]; echo $AUTOPKGTEST_TMP\n[ "${AUTOPKGTEST_TMP#/var/tmp}" != "$AUTOPKGTEST_TMP" ]'})

        env = os.environ.copy()
        env['TMPDIR'] = '/var/tmp'
        (code, out, err) = self.runtest(['--no-built-binaries', p],
                                        env=env)

        # test results
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'/var/tmp/autopkgtest.*')

    def test_build_parallel(self):
        '''--build-parallel option'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: build-needed\n',
                           {'pass': '#!/bin/sh -e\necho DBO=$DEB_BUILD_OPTIONS\n'})

        (code, out, err) = self.runtest(['--build-parallel=17', '--no-built-binaries', p])
        # test should succeed
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should build package
        self.assertIn('dh build', err)

        # should have expected parallel= option
        self.assertIn('DBO=parallel=17\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_setup_commands_fail_transient(self):
        '''--setup-commands which exit with 1 (transient error)'''

        p = self.build_src('Test-Command: false\nDepends:\n', {})

        (code, out, err) = self.runtest(['-B', '-d', p, '--setup-commands', 'false'])

        self.assertEqual(code, 16, err)
        self.assertNotIn('badpkg', out)
        self.assertIn('ERROR: testbed failure: testbed setup commands failed with status 1', err)
        self.assertNotIn('command1', out)

    def test_setup_commands_boot_fail_transient(self):
        '''--setup-commands-boot which exit with 1 (transient error)'''

        p = self.build_src('Test-Command: false\nDepends:\n', {})

        (code, out, err) = self.runtest(['-B', '-d', p, '--setup-commands-boot', 'false'])

        self.assertEqual(code, 16, err)
        self.assertNotIn('badpkg', out)
        self.assertIn('ERROR: testbed failure: testbed boot setup commands failed with status 1', err)
        self.assertNotIn('command1', out)

    def test_setup_commands_fail_badpkg(self):
        '''--setup-commands which exit with 100 (bad package)'''

        p = self.build_src('Test-Command: false\nDepends:\n', {})

        (code, out, err) = self.runtest(['-B', '-d', p, '--setup-commands', 'exit 100'])

        self.assertEqual(code, 12, err)
        self.assertIn('badpkg: testbed setup commands failed with status 100', out)
        self.assertNotIn('command1', out)

    def test_setup_commands_boot_fail_badpkg(self):
        '''--setup-commands-boot which exit with 100 (bad package)'''

        p = self.build_src('Test-Command: false\nDepends:\n', {})

        (code, out, err) = self.runtest(['-B', '-d', p, '--setup-commands-boot', 'exit 100'])

        self.assertEqual(code, 12, err)
        self.assertIn('badpkg: testbed boot setup commands failed with status 100', out)
        self.assertNotIn('command1', out)

    def test_reboot(self):
        '''needs-reboot test gets skipped'''

        p = self.build_src('Test-Command: true\nDepends:\nRestrictions: needs-reboot', {})
        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'command1\s+SKIP Test needs to reboot testbed but testbed does not provide reboot capability')

    def test_reboot_mixed(self):
        '''needs-reboot test gets skipped but another test is still run'''

        p = self.build_src('Test-Command: true 1\nDepends:\nRestrictions: needs-reboot\n'
                           '\n'
                           'Test-Command: true 2\nDepends:\n',
                           {})
        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, r'command1\s+SKIP Test needs to reboot testbed but testbed does not provide reboot capability')
        self.assertRegex(out, r'command2\s+PASS')

    @unittest.skipIf(os.getuid() == 0, 'failure mode for root is different and tested elsewhere')
    def test_broken_test_deps(self):
        '''unsatisfiable test dependencies'''

        p = self.build_src('Tests: p\nDepends: unknown, libc6 (>= 99:99)\n',
                           {'p': '#!/bin/sh -e\ntrue'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 12, err)
        self.assertIn('test dependencies missing', err)
        self.assertIn('test dependencies missing', out)

    def test_continue_with_other_tests(self):
        '''Install failure should continue with next test'''

        p = self.build_src('Test-Command: false\nDepends:\n\nTest-Command: true\nDepends:unknown\n\nTest-Command: true\nDepends:\n', {})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--debug', '--no-built-binaries', p, '--summary=' + summary])
        self.assertEqual(code, 12, err)
        self.assertRegex(out, r'command1\s+FAIL')
        self.assertRegex(out, r'command2\s+FAIL')
        self.assertRegex(out, r'command3\s+PASS')

    def test_skip_not_installable(self):
        '''Test skip-not-installable restriction'''
        p = self.build_src('Tests: notinstallable\n'
                           'Restrictions: skip-not-installable\n'
                           'Depends: nonexisting\n',
                           {'notinstallable': '#!/bin/sh\necho But I am fine\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'notinstallable\s+SKIP installation fails', out)

    def test_skip_not_installable_mixed(self):
        '''Test skip-not-installable restriction combined with a passing test'''
        p = self.build_src('Tests: notinstallable\n'
                           'Restrictions: skip-not-installable\n'
                           'Depends: nonexisting\n'
                           '\n'
                           'Tests: ok\n'
                           'Depends:\n',
                           {'notinstallable': '#!/bin/sh\necho But I am fine\n',
                            'ok': '#!/bin/sh\nexit 0\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, r'notinstallable\s+SKIP installation fails', out)
        self.assertRegex(out, r'ok\s+PASS', out)

    def test_validate(self):
        '''--validate command line options'''
        p = self.build_src('Tests: hello-world\n'
                           'Depends: coreutils\n',
                           {'hello-world': '#!/bin/sh\necho "HELLO WORLD"\n'})
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', '--validate', p])
        self.assertEqual(code, 0, err)
        self.assertIn('Test specification is valid', out)
        self.assertNotIn('HELLO WORLD', out)

    def test_unknown_restriction(self):
        '''test with unknown restriction gets skipped'''

        p = self.build_src('Test-Command: true\nDepends:\nRestrictions: needs-reassurance', {})
        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'command1\s+SKIP unknown restriction needs-reassurance')

    def test_unknown_derestriction(self):
        '''--ignore-restrictions is respected for unknown restrictions'''

        p = self.build_src('Test-Command: true\nDepends:\nRestrictions: needs-reassurance', {})
        (code, out, err) = self.runtest(['-d', '-B', '--ignore-restrictions=needs-reassurance', p])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'command1\s+PASS', out)

    def test_known_derestriction(self):
        '''--ignore-restrictions is respected for known restrictions'''

        p = self.build_src('Test-Command: true\nDepends:\nRestrictions: needs-reboot', {})
        (code, out, err) = self.runtest(['-d', '-B', '--ignore-restrictions=needs-reboot', p])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'command1\s+PASS', out)


@unittest.skipIf(os.getuid() > 0,
                 'NullRunnerRoot tests need to run as root')
class NullRunnerRoot(AdtTestCase):
    def __init__(self, *args, **kwargs):
        super(NullRunnerRoot, self).__init__(['null'], *args, **kwargs)

    def test_broken_test_deps(self):
        '''unsatisfiable test dependencies'''

        p = self.build_src('Tests: p\nDepends: unknown, libc6 (>= 99:99)\n',
                           {'p': '#!/bin/sh -e\ntrue'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 12, err)
        self.assertIn('Test dependencies are unsatisfiable', err)
        self.assertIn('Test dependencies are unsatisfiable', out)

    def test_tmpdir_for_other_users(self):
        '''$TMPDIR is accessible to non-root users'''

        prev_mask = os.umask(0o077)
        self.addCleanup(os.umask, prev_mask)
        p = self.build_src('Tests: t\nDepends: coreutils\nRestrictions: needs-root\n',
                           {'t': '''#!/bin/sh -e
echo hello > ${TMPDIR:=/tmp}/rootowned.txt
su -s /bin/sh -c "echo hello > $TMPDIR/world.txt" nobody
if su -s /bin/sh -c "echo break > $TMPDIR/rootowned.txt" nobody 2>/dev/null; then
    exit 1
fi
rm $TMPDIR/rootowned.txt $TMPDIR/world.txt
'''})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # test should succeed
        self.assertRegex(out, r't\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_user(self):
        '''--user option'''

        p = self.build_src('Tests: t\nDepends:\n',
                           {'t': '''#!/bin/sh -e
                                    echo world > ${TMPDIR:=/tmp}/hello.txt
                                    cat $TMPDIR/hello.txt
                                    rm $TMPDIR/hello.txt
                                    stat -c %U .
                                    stat -c %U debian
                                    stat -c %U Makefile
                                    echo "USER: $USER"
                                    whoami'''})

        (code, out, err) = self.runtest(['--no-built-binaries', p, '--user=nobody'])
        self.assertEqual(code, 0, err)

        # test should succeed
        self.assertRegex(out, r't\s+PASS', out)

        # has output from cat and whoami
        self.assertIn('world\nnobody\nnobody\nnobody\nUSER: nobody\nnobody\n', out)

    def test_default_locale(self):
        '''tests have default locale C.UTF-8'''

        p = self.build_src('Tests: r\nDepends:\nRestrictions: needs-root\n\n'
                           'Tests: u\nDepends:\n',
                           {'r': '#!/bin/sh -e\necho root $LANG\nlocale',
                            'u': '#!/bin/sh -e\necho user $LANG\nlocale'})

        # invalid LC_* should not be copied, only $LANG set
        env = os.environ.copy()
        env['LC_PAPER'] = 'fo_BAR.UTF-9'
        (code, out, err) = self.runtest(['-B', p], env=env)
        self.assertEqual(code, 0, err)

        # test should succeed
        self.assertRegex(out, r'r\s+PASS', out)
        self.assertRegex(out, r'u\s+PASS', out)

        # has expected locales
        self.assertIn('root C.UTF-8\n', out)
        self.assertIn('user C.UTF-8\n', out)

    def test_specify_locale(self):
        '''tests have specified locale'''

        p = self.build_src('Tests: r\nDepends:\nRestrictions: needs-root\n\n'
                           'Tests: u\nDepends:\n',
                           {'r': '#!/bin/sh -e\necho root $LANG',
                            'u': '#!/bin/sh -e\necho user $LANG'})

        (code, out, err) = self.runtest(['-B', p, '--set-lang=ab_CD.UTF-8'])
        self.assertEqual(code, 0, err)

        # test should succeed
        self.assertRegex(out, r'r\s+PASS', out)
        self.assertRegex(out, r'u\s+PASS', out)

        # has expected locales
        self.assertIn('root ab_CD.UTF-8\n', out)
        self.assertIn('user ab_CD.UTF-8\n', out)


@unittest.skipIf('cowdancer' in os.environ.get('LD_PRELOAD', ''),
                 'chroot tests do not work under cowdancer')
@unittest.skipIf(os.getuid() > 0,
                 'chroot runner needs to run as root')
class ChrootRunner(AdtTestCase, DebTestsAll, DebTestsFailureModes):
    def __init__(self, *args, **kwargs):
        super(ChrootRunner, self).__init__(['chroot', '/uninited'], *args, **kwargs)

    def setUp(self):
        super(ChrootRunner, self).setUp()

        def install_file(path):
            destdir = self.chroot + '/' + os.path.dirname(path)
            if not os.path.exists(destdir):
                os.makedirs(destdir)
            if os.path.isfile(path):
                shutil.copy(path, destdir)
            else:
                subprocess.check_call(['cp', '-a', path, destdir])

        def install_elf(path):
            install_file(path)
            out = subprocess.check_output(['ldd', path], universal_newlines=True)
            libs = set()
            for lib in re.finditer('/[^ ]+', out):
                libs.add(lib.group(0))
            for lib in libs:
                install_file(lib)

        # build a mini-chroot
        self.chroot = os.path.join(self.workdir, 'chroot')

        install_file('/usr/bin/which')
        install_elf('/bin/bash')
        install_elf('/bin/sh')
        install_elf('/bin/ls')
        install_elf('/bin/cat')
        install_elf('/bin/rm')
        install_elf('/bin/cp')
        install_elf('/bin/mkdir')
        install_elf('/bin/chmod')
        install_elf('/bin/chown')
        install_elf('/bin/mktemp')
        install_elf('/bin/tar')
        install_elf('/bin/gzip')
        install_elf('/bin/sleep')
        install_elf('/bin/sed')
        install_elf('/bin/readlink')
        install_elf('/bin/grep')
        install_elf('/bin/uname')
        install_elf('/usr/bin/test')
        install_elf('/usr/bin/awk')
        install_elf('/usr/bin/head')
        install_elf('/usr/bin/env')
        install_elf('/usr/bin/tee')
        install_elf('/usr/bin/touch')
        install_elf('/usr/bin/nproc')
        install_file('/usr/lib/locale/C.UTF-8')

        # necessary bind mounts
        dev_dir = os.path.join(self.chroot, 'dev')
        os.mkdir(dev_dir)
        subprocess.check_call(['mount', '-o', 'bind', '/dev', dev_dir])
        proc_dir = os.path.join(self.chroot, 'proc')
        os.mkdir(proc_dir)
        subprocess.check_call(['mount', '-o', 'bind', '/proc', proc_dir])

        # some fakes
        for cmd in ['dpkg', 'dpkg-query', 'dpkg-source', 'apt-get', 'apt-key', 'apt-cache']:
            p = os.path.join(self.chroot, 'usr', 'bin', cmd)
            with open(p, 'w') as f:
                f.write('#!/bin/sh -e\n')
                if cmd == 'dpkg':
                    f.write('if [ "$1" = "--print-architecture" ]; then echo powerpc; exit; fi\n')
                f.write('echo "fake-%s: $@"\n' % cmd)
                if cmd == 'apt-get':
                    f.write('if [ "$1" = source ]; then cp /testpkg_* .; fi\n')
                if cmd == 'apt-cache':
                    f.write('if [ "$1" = showsrc ]; then printf "Package-List:\\n $3 deb utils optional arch=any\\nFormat: 1.0\\n"; fi\n')
                if cmd == 'dpkg-source':
                    f.write('if [ "$1" = "-x" ]; then mkdir "$3"; tar -C "$3" --strip-components=1 -xf ${2%.dsc}.tar.*; fi\n')
            os.chmod(p, 0o755)
        with open(os.path.join(self.chroot, 'bin', 'su'), 'w') as f:
            f.write('#!/bin/sh\nshift 4\nexec bash -c "$@"\n')
            os.fchmod(f.fileno(), 0o755)
        with open(os.path.join(self.chroot, 'usr', 'bin', 'id'), 'w') as f:
            f.write('#!/bin/sh\necho root\n')
            os.fchmod(f.fileno(), 0o755)

        p = os.path.join(self.chroot, 'tmp')
        os.mkdir(p)
        os.chmod(p, 0o177)
        os.makedirs(os.path.join(self.chroot, 'etc', 'apt', 'sources.list.d'))

        self.virt_args[-1] = self.chroot

    def tearDown(self):
        # these sometimes fail on EBUSY
        subprocess.call('for p in %(c)s/dev %(c)s/proc; do while mountpoint $p >/dev/null; do umount $p; sleep 0.1; done; done' %
                        {'c': self.chroot}, shell=True)
        subprocess.call('for i in `seq 10`; do rm -rf %s && break; sleep 0.5; done' % self.chroot, shell=True)

    def test_fancy_deps(self):
        '''wrapped and versioned test dependencies'''

        p = self.build_src('''Tests: pass
Depends: fancypkg,
         coreutils | vanilla (>= 10),
         chocolate,
Restrictions: needs-root
''', {'pass': '#!/bin/sh\ntrue'})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS')

        self.assertIn('processing dependency fancypkg\n', err)
        self.assertIn('processing dependency coreutils | vanilla (>= 10)\n', err)
        self.assertIn('processing dependency chocolate\n', err)

    def test_build_deps(self):
        '''test depends on build dependencies'''

        p = self.build_src('''Tests: pass
Depends: @, testdep1, @builddeps@, testdep2,
# blabla ☺
 testdep3
Restrictions: needs-root
''', {'pass': '#!/bin/sh\ntrue'})

        # add extra build dependencies to testpkg
        subprocess.check_call(['sed', '-i', '/^Build-Depends:/ s/:.*/: bdep1 , bdep2,\\n bdep3, # moo\\n#comment\\n bdep4\\n'
                                            'Build-Depends-Indep: bdep5\\n'
                                            'Build-Depends-Arch: bdep6/',
                               os.path.join(p, 'debian', 'control')])

        # run this under C locale to test that UTF-8 debian/control is still
        # handled correctly
        (code, out, err) = self.runtest(['-d', '-B', p],
                                        env={'LC_ALL': 'C', 'PATH': os.environ['PATH']})
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS')

        self.assertIn('synthesised dependency testpkg\n', err)
        self.assertIn('processing dependency testdep1\n', err)
        self.assertIn('synthesised dependency bdep1\n', err)
        self.assertIn('synthesised dependency bdep2\n', err)
        self.assertIn('synthesised dependency bdep3\n', err)
        self.assertIn('synthesised dependency bdep4\n', err)
        self.assertIn('synthesised dependency bdep5\n', err)
        self.assertIn('synthesised dependency bdep6\n', err)
        self.assertIn('synthesised dependency build-essential\n', err)
        self.assertIn('processing dependency testdep2\n', err)

    def test_build_deps_profiles(self):
        '''test depends on build dependencies with build profiles'''

        p = self.build_src('''Tests: pass
Depends: @builddeps@
Restrictions: needs-root
''', {'pass': '#!/bin/sh\ntrue'})

        # add extra build dependencies to testpkg
        subprocess.check_call(['sed', '-i', '/^Build-Depends:/ s/:.*/: bdepyes <!nocheck>, bdepno <stage1> <cross>/',
                               os.path.join(p, 'debian', 'control')])

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS')

        self.assertIn('synthesised dependency bdepyes\n', err)
        self.assertIn('synthesised dependency build-essential\n', err)

        dpkg_deps_ver = subprocess.check_output(['perl', '-MDpkg::Deps', '-e', 'print $Dpkg::Deps::VERSION'],
                                                universal_newlines=True)
        if dpkg_deps_ver >= '1.04':
            self.assertNotIn('bdepno', err)

    @unittest.skip('chroot runner tests do not support building')
    def test_tree_build_needed_success(self):
        pass

    def test_artifacts(self):
        '''tests producing additional artifacts'''

        p = self.build_src('Tests: a1 a2 a3 a4\nDepends:\nRestrictions: needs-root\n',
                           {'a1': '#!/bin/sh -e\n[ -d "$AUTOPKGTEST_ARTIFACTS" ]\n'
                                  'echo old > $AUTOPKGTEST_ARTIFACTS/health.txt\n',
                            'a2': '#!/bin/sh -e\n[ -d "$AUTOPKGTEST_ARTIFACTS" ]\n'
                                  'echo I am fine > $AUTOPKGTEST_ARTIFACTS/health.txt\n',
                            'a3': '#!/bin/sh -e\n[ -d "$AUTOPKGTEST_ARTIFACTS" ]\n'
                                  'mkdir $AUTOPKGTEST_ARTIFACTS/logs\n'
                                  'echo world > $AUTOPKGTEST_ARTIFACTS/logs/hello.txt\n',
                            # use obsolete $ADT_ARTIFACTS here to verify backwards compat
                            'a4': '#!/bin/sh -e\n[ -d "$ADT_ARTIFACTS" ]\n'
                                  'mkdir $ADT_ARTIFACTS/logs\n'
                                  'echo 42 > $ADT_ARTIFACTS/logs/answer.txt\n'})

        outdir = os.path.join(self.workdir, 'out')

        (code, out, err) = self.runtest(['--no-built-binaries', p, '--output-dir=' + outdir])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'a1\s+PASS', out)
        self.assertRegex(out, r'a2\s+PASS', out)
        self.assertRegex(out, r'a3\s+PASS', out)
        self.assertRegex(out, r'a4\s+PASS', out)

        # check for cruft in output dir
        files = [i for i in os.listdir(outdir)
                 if not fnmatch.fnmatch(i, 'a*-std*') and
                 not fnmatch.fnmatch(i, 'a*-packages')]
        self.assertEqual(set(files),
                         set(['log', 'artifacts', 'testpkg-version',
                              'testbed-packages', 'summary', 'testinfo.json']))

        # check artifact; a2 should overwrite a1's health.txt
        with open(os.path.join(outdir, 'artifacts', 'health.txt')) as f:
            self.assertEqual(f.read(), 'I am fine\n')
        with open(os.path.join(outdir, 'artifacts', 'logs', 'hello.txt')) as f:
            self.assertEqual(f.read(), 'world\n')
        with open(os.path.join(outdir, 'artifacts', 'logs', 'answer.txt')) as f:
            self.assertEqual(f.read(), '42\n')

    def test_slash_in_test_name(self):
        '''test names must not contain /'''

        p = self.build_src('Tests: pass subdir/p\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\ntrue'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # invalid test gets skipped
        self.assertEqual(code, 2, err)
        self.assertRegex(out, r'subdir/p\s+SKIP test name may not contain /.*')

        # valid test still gets run
        self.assertFalse(re.match(r'pass\s+SKIP', out), out)
        self.assertRegex(out, r'pass\s+PASS', out)

    def test_unicode(self):
        '''Unicode test output'''

        p = self.build_src('Tests: se\nDepends:\nRestrictions: needs-root\n',
                           {'se': '#!/bin/sh\necho ‘a♩’; echo fancy ‴ʎɔuɐɟ″!>&2'})

        summary = os.path.join(self.workdir, 'summary.log')

        (code, out, err) = self.runtest(['--no-built-binaries', p, '--summary=' + summary])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'se\s+FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

        # should show test stdout/stderr
        self.assertRegex(out, r'‘a♩’\n')
        self.assertRegex(err, r'\nfancy ‴ʎɔuɐɟ″!\n')

        with open(summary, encoding='UTF-8') as f:
            self.assertRegex(f.read(), r'se\s+FAIL stderr: fancy ‴ʎɔuɐɟ″!\n')

    def test_apt_source_no_restrictions(self):
        '''apt source, no build, no restrictions'''

        p = self.build_dsc('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})
        # copy that into the chroot where fake apt/dpkg can find it
        shutil.copy(p, self.chroot)
        shutil.copy(p.replace('.dsc', '.tar.gz'), self.chroot)

        (code, out, err) = self.runtest(['-d', 'testpkg'])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)I am fine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not build package
        self.assertNotIn('dh build', err)

    def test_apt_source_single_line_package_list(self):
        '''apt source, single-line Package-List:'''

        # modify fake apt-cache to show single-line P-L
        with open(os.path.join(self.chroot, 'usr', 'bin', 'apt-cache'), 'w') as f:
            f.write('if [ "$1" = showsrc ]; then printf "Package-List: $3 deb utils optional arch=any\\n"; fi\n')

        p = self.build_dsc('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})
        # copy that into the chroot where fake apt-get source can find it
        shutil.copy(p, self.chroot)
        shutil.copy(p.replace('.dsc', '.tar.gz'), self.chroot)

        (code, out, err) = self.runtest(['-d', 'testpkg'])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

    def test_setup_commands_string(self):
        '''--setup-commands with command string'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\n[ -x /bin/cp_cp ]; cat /setup.log\n'})

        # not expecting a normal user for chroot; but filter out any
        # $AUTOPKGTEST_NORMAL_USER from *our* environment
        env = os.environ.copy()
        try:
            del env['AUTOPKGTEST_NORMAL_USER']
        except KeyError:
            pass

        # not expecting a normal user for chroot
        (code, out, err) = self.runtest(['-B', p,
                                         '--setup-commands', '[ -z $AUTOPKGTEST_NORMAL_USER ];'
                                         'sleep 3; cp /bin/cp /bin/cp_cp; '
                                         'echo setup_success > /setup.log',
                                         '--setup-commands', 'cp /bin/cp /bin/cp_cp',
                                         '--timeout-short=1', '--timeout-copy=1'], env=env)
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'^setup_success\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        self.assertIn('@@@@@@ test bed setup', err)

    def test_setup_commands_file(self):
        '''--setup-commands with command file'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\n[ -x /bin/cp_cp ]; cat /setup.log\ncat /s2.log'})

        cmds = os.path.join(self.workdir, 'setup.sh')
        with open(cmds, 'w') as f:
            f.write('cp /bin/cp /bin/cp_cp\necho setup_success > /setup.log\n')
            f.flush()
        cmds2 = os.path.join(self.workdir, 'setup2.sh')
        with open(cmds2, 'w') as f:
            f.write('echo setup2_success > /s2.log\n')
            f.flush()

        (code, out, err) = self.runtest(['-B', '-d', p,
                                         '--setup-commands', cmds,
                                         '--setup-commands', cmds2])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'^setup_success\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        self.assertIn('@@@@@@ test bed setup', err)

    def test_copy(self):
        '''--copy'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ncp -a /mytestdata/* $AUTOPKGTEST_ARTIFACTS\necho test_ok'})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['-B', p, '-o', outdir,
                                         '--copy', '%s:/mytestdata/Makefile' % os.path.join(root_dir, 'Makefile'),
                                         '--copy', '%s:/mytestdata/tree' % os.path.join(test_dir, 'testpkg')])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertRegex(out, r'^test_ok\n')

        # has copied the file correctly
        f = os.path.join(outdir, 'artifacts', 'Makefile')
        self.assertTrue(os.path.isfile(f))
        with open(f) as fcopy:
            with open(os.path.join(root_dir, 'Makefile')) as forig:
                self.assertEqual(forig.read(), fcopy.read())

        # has copied the dir correctly
        d = os.path.join(outdir, 'artifacts', 'tree')
        self.assertTrue(os.path.isdir(d))
        subprocess.check_call(['diff', '-Nur', os.path.join(test_dir, 'testpkg'), d])

    def test_apt_pocket(self):
        '''--apt-pocket'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ntest -e /etc/apt/sources.list.d/proposed.list'})

        apt_dir = os.path.join(self.chroot, 'etc', 'apt')
        with open(os.path.join(apt_dir, 'sources.list'), 'w') as f:
            f.write('''# comment
deb http://foo.ubuntu.com/ fluffy-updates main non-free
deb-src http://foo.ubuntu.com/ fluffy-updates main non-free
deb http://foo.ubuntu.com/ fluffy main non-free
deb-src http://foo.ubuntu.com/ fluffy main non-free
deb http://bar.debian.org/ fluffy extras
deb-src http://bar.debian.org/ fluffy extras
# third-party repo
deb http://something.else.net/ fluffy addons
# options
deb [trusted=yes arch=6510] http://foo.ubuntu.com/ fluffy main 6510
''')
        with open(os.path.join(apt_dir, 'sources.list.d', 'restricted.list'), 'w') as f:
            f.write('deb http://foo.ubuntu.com/ fluffy restricted\n')

        (code, out, err) = self.runtest(['-B', '-d', p, '--apt-pocket', 'proposed'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # verify proposed.list
        with open(os.path.join(apt_dir, 'sources.list.d', 'proposed.list')) as f:
            self.assertEqual(f.read(), '''deb http://foo.ubuntu.com/ fluffy-proposed main non-free
deb-src http://foo.ubuntu.com/ fluffy-proposed main non-free
deb http://bar.debian.org/ fluffy-proposed extras
deb-src http://bar.debian.org/ fluffy-proposed extras
deb http://something.else.net/ fluffy-proposed addons
deb [trusted=yes arch=6510] http://foo.ubuntu.com/ fluffy-proposed main 6510
deb http://foo.ubuntu.com/ fluffy-proposed restricted
''')
        # should not have any apt pinning
        self.assertFalse(os.path.exists(os.path.join(apt_dir, 'preferences.d')))

    def test_apt_pocket_selected(self):
        '''--apt-pocket with package list'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ntest -e /etc/apt/sources.list.d/proposed.list'})

        apt_dir = os.path.join(self.chroot, 'etc', 'apt')
        with open(os.path.join(apt_dir, 'sources.list'), 'w') as f:
            f.write('''# comment
deb http://foo.ubuntu.com/ fluffy main non-free
deb-src http://foo.ubuntu.com/ fluffy main non-free
deb http://foo.ubuntu.com/ fluffy-updates main non-free
deb-src http://foo.ubuntu.com/ fluffy-updates main non-free
deb [trusted=yes arch=6510] http://foo.ubuntu.com/ fluffy main 6510
''')

        (code, out, err) = self.runtest(['-B', '-d', p, '--apt-pocket', 'proposed=foo,bar'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # verify proposed.list
        with open(os.path.join(apt_dir, 'sources.list.d', 'proposed.list')) as f:
            self.assertEqual(f.read(), '''deb http://foo.ubuntu.com/ fluffy-proposed main non-free
deb-src http://foo.ubuntu.com/ fluffy-proposed main non-free
deb [trusted=yes arch=6510] http://foo.ubuntu.com/ fluffy-proposed main 6510
''')

        # should set up apt pinning
        self.assertEqual(sorted(os.listdir(os.path.join(apt_dir, 'preferences.d'))),
                         ['autopkgtest-default-release', 'autopkgtest-fluffy-proposed'])
        with open(os.path.join(apt_dir, 'preferences.d', 'autopkgtest-fluffy-proposed')) as f:
            self.assertEqual(f.read(), '''Package: foo bar
Pin: release a=fluffy-proposed
Pin-Priority: 995

Package: *
Pin: release a=fluffy-updates
Pin-Priority: 990
''')
        with open(os.path.join(apt_dir, 'preferences.d', 'autopkgtest-default-release')) as f:
            self.assertEqual(f.read(), '''Package: *
Pin: release a=fluffy
Pin-Priority: 990
''')

    def test_apt_default_release(self):
        '''--apt-default-release'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ntest -e /etc/apt/preferences.d/autopkgtest-default-release'})

        (code, out, err) = self.runtest(['-B', '-d', p, '--apt-default-release', 'testing'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        apt_dir = os.path.join(self.chroot, 'etc', 'apt')

        # should set up pinning to get Default-Release
        self.assertEqual(os.listdir(os.path.join(apt_dir, 'preferences.d')),
                         ['autopkgtest-default-release'])
        with open(os.path.join(apt_dir, 'preferences.d', 'autopkgtest-default-release')) as f:
            self.assertEqual(f.read(), '''Package: *
Pin: release testing
Pin-Priority: 990
''')

    def test_add_apt_source(self):
        '''--add-apt-source'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ntest -e /etc/apt/sources.list.d/autopkgtest-add-apt-sources.list'})

        apt_dir = os.path.join(self.chroot, 'etc', 'apt')

        (code, out, err) = self.runtest(['-B', '-d', p, '--add-apt-source', 'deb http://foo.ubuntu.com/ fluffy-proposed main'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # verify proposed.list
        with open(os.path.join(apt_dir, 'sources.list.d', 'autopkgtest-add-apt-sources.list')) as f:
            self.assertEqual(f.read(), 'deb http://foo.ubuntu.com/ fluffy-proposed main\n')

        # should not have any apt pinning
        self.assertFalse(os.path.exists(os.path.join(apt_dir, 'preferences.d')))

    def test_add_apt_release(self):
        '''--add-apt-release'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ntest -e /etc/apt/sources.list.d/autopkgtest-add-apt-release-fluffyplusone.list'})

        apt_dir = os.path.join(self.chroot, 'etc', 'apt')
        with open(os.path.join(apt_dir, 'sources.list'), 'w') as f:
            f.write('''# comment
deb http://foo.ubuntu.com/ fluffy main non-free
''')

        (code, out, err) = self.runtest(['-B', '-d', p, '--add-apt-release', 'fluffyplusone'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # verify new sources.list
        with open(os.path.join(apt_dir, 'sources.list.d', 'autopkgtest-add-apt-release-fluffyplusone.list')) as f:
            self.assertEqual(f.read(), 'deb http://foo.ubuntu.com/ fluffyplusone main non-free\ndeb-src http://foo.ubuntu.com/ fluffyplusone main non-free\n')

    def test_pin_packages(self):
        '''--pin-packages'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh -e\ntest -e /etc/apt/preferences.d/autopkgtest-fluffy-proposed'})

        apt_dir = os.path.join(self.chroot, 'etc', 'apt')
        with open(os.path.join(apt_dir, 'sources.list'), 'w') as f:
            f.write('''# comment
deb http://foo.ubuntu.com/ fluffy main non-free
''')

        (code, out, err) = self.runtest(['-B', '-d', p, '--pin-packages', 'fluffy-proposed=foo,bar'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should set up apt pinning
        self.assertEqual(sorted(os.listdir(os.path.join(apt_dir, 'preferences.d'))),
                         ['autopkgtest-default-release', 'autopkgtest-fluffy-proposed'])
        with open(os.path.join(apt_dir, 'preferences.d', 'autopkgtest-fluffy-proposed')) as f:
            self.assertEqual(f.read(), '''Package: foo bar
Pin: release a=fluffy-proposed
Pin-Priority: 995
''')

        # should set up pinning for Default-Release
        with open(os.path.join(apt_dir, 'preferences.d', 'autopkgtest-default-release')) as f:
            self.assertEqual(f.read(), '''Package: *
Pin: release fluffy
Pin-Priority: 990
''')

    def test_tree_garbage(self):
        '''copied source tree contains only expected files'''

        p = self.build_src('Tests: g\nDepends:\nRestrictions: needs-root\n',
                           {'g': '#!/bin/sh\npwd\nLC_ALL=C ls .\n'})

        (code, out, err) = self.runtest(['-B', p])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'g\s+PASS', out)

        self.assertRegex(out, r'/autopkgtest.*/real-tree\n'
                         'Makefile\ndebian\ntest_static\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_no_tests_dir(self):
        '''package without debian/tests/'''

        p = self.build_src(None, None)
        (code, out, err) = self.runtest(['-B', p])
        self.assertEqual(code, 8, err)
        self.assertRegex(out, r'SKIP no tests in this package', out)

    def test_override_control(self):
        '''custom control file path with --override-control'''

        p = self.build_src('Tests: NOTME\nDepends: nonexisting\nRestrictions: breaks-testbed\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        custom_control = os.path.join(self.workdir, 'myctrl')
        with open(custom_control, 'w') as f:
            f.write('Tests: pass\nDepends:\nRestrictions: needs-root')

        (code, out, err) = self.runtest(['-B', '--override-control=' + custom_control, p])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'^I am fine\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not try anything with the bad test
        self.assertNotIn('NOTME', out)
        self.assertNotIn('NOTME', err)

    def test_nonexisting_override_control(self):
        '''nonexisting --override-control'''

        p = self.build_src('Tests: NOTME\nDepends: nonexisting\n', {})
        (code, out, err) = self.runtest(['--override-control=/nonexisting', p])
        self.assertEqual(code, 20, err)
        self.assertIn('/nonexisting', err)

        # should not try anything with the original control file
        self.assertNotIn('NOTME', out)
        self.assertNotIn('NOTME', err)

    def test_relative_chroot_path(self):
        '''relative chroot path'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        os.chdir(self.workdir)
        (code, out, err) = self.runtest(['--no-built-binaries', p],
                                        virt_args=['chroot', os.path.basename(self.chroot)])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)
        # should show test stdout
        self.assertRegex(out, r'^I am fine\n')

    def test_cli_args_from_file(self):
        '''read CLI arguments from file'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root\n',
                           {'pass': '#!/bin/sh\necho I am fine\n'})

        argfile = os.path.join(self.workdir, 'myopts')
        with open(argfile, 'w') as f:
            f.write(' -d \n%s' % self.chroot)

        (code, out, err) = self.runtest(['--no-built-binaries', p],
                                        virt_args=['chroot', '@' + argfile])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should have debug messages
        self.assertIn('chroot: DBG:', err)

    def test_no_dpkg_query(self):
        '''testbed does not have dpkg-query'''

        p = self.build_src('Tests: t\nDepends:\nRestrictions: needs-root\n',
                           {'t': '#!/bin/sh -e\necho ok\n'})
        outdir = os.path.join(self.workdir, 'out')
        (code, out, err) = self.runtest(['--setup-commands', 'rm /usr/bin/dpkg-query',
                                         '-d', '-B', p, '--output-dir=' + outdir])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r't\s+PASS', out)

        # check outdir
        with open(os.path.join(outdir, 't-stdout')) as f:
            self.assertEqual(f.read(), 'ok\n')
        with open(os.path.join(outdir, 'log')) as f:
            contents = f.read()
        self.assertRegex(contents, r't\s+PASS\n')

        # don't create empty/bogus files
        self.assertNotIn('testbed-packages', os.listdir(outdir))

    @unittest.skip('chroot runner tests fake testbed_arch as powerpc')
    def test_arch_in_mixed_list_skipped_explicit(self):
        pass

    @unittest.skip('chroot runner tests fake testbed_arch as powerpc')
    def test_arch_in_unsupported_list(self):
        pass

    @unittest.skip('chroot runner tests fake testbed_arch as powerpc')
    def test_arch_in_supported_list(self):
        pass


class DebTestsVirtFS(DebTestsAll):
    '''Common tests for runners with file system virtualization'''

    def test_apt_autodep8_with_control(self):
        '''autodep8 with d/t/control present'''

        p = self.build_src('Tests: p\nDepends: perl-base\n',
                           {'p': '#!/bin/sh -e\ntrue'})

        # add a Testsuite
        subprocess.check_call(['sed', '-i', r'/^Build-Depends:/ a\Testsuite: autopkgtest-pkg-perl',
                               os.path.join(p, 'debian', 'control')])

        (code, out, err) = self.runtest([p])
        self.assertEqual(code, 0, err)
        self.assertIn('test p:', err)
        self.assertIn('/usr/share/pkg-perl-autopkgtest/runner build-deps', err)

    def test_tree_norestrictions_nobuild_success(self):
        '''source tree, no build, no restrictions, test success'''

        # also test some fancy dependencies
        p = self.build_src('Tests: p1 p2\n'
                           'Depends: coreutils, aspell-doc [linux-any], apt-doc:native, unknown [nosucharch], bash-doc [!c64], bogus [!linux-any]\n'
                           'Restrictions: needs-root\n',
                           {'p1': '#!/bin/sh -e\nif dpkg -s autopkgtest-satdep 2>/dev/null; then exit 1; fi; '
                            'apt-get --purge -y autoremove; '
                            'ls /usr/share/doc/aspell-doc/copyright;'
                            'echo I am fine\n',
                            'p2': '#!/bin/sh\necho I am also fine; echo "SOMEVAR: >$SOMEVAR<"; echo "PATH: $PATH"\n'})

        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p],
                                        env={'SOMEVAR': 'junk',
                                             'HOME': os.environ['HOME'],
                                             'PATH': '/something/broken:' + os.environ.get('PATH', '')})

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'p1\s+PASS', out)
        self.assertRegex(out, r'p2\s+PASS', out)

        # $SOMEVAR does not leak into container
        self.assertIn('\nSOMEVAR: ><\n', out)
        # modified $PATH does not leak into container
        self.assertNotIn('/something/broken', out)

        # handles expected packages
        self.assertIn('processing dependency coreutils', err)
        self.assertIn('Unpacking aspell-doc', out)
        self.assertIn('Unpacking apt-doc', out)
        self.assertIn('Unpacking bash-doc', out)
        self.assertNotIn('unknown', out)
        self.assertNotIn('bogus', out)
        self.assertNotIn('Removing aspell-doc', out)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)I am fine\n')
        self.assertIn('\nI am also fine\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not build package
        self.assertNotIn('dh build', err)

        # should not show boot messages
        self.assertNotIn(' login:', err)

    def test_tree_built_binaries(self):
        '''source tree, install built binaries'''

        # no "build-needed" restriction here, it should be built because we
        # need to install the package as a dependency (@)
        p = self.build_src('Tests: pass\n',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built | grep -q "built script OK"\necho GOOD'})

        # add extra dependency to testpkg, to ensure autopkgtest does not stop
        # and ask for confirmation in apt-get
        subprocess.check_call(['sed', '-i', '/^Depends:/ s/$/, aspell-doc/',
                               os.path.join(p, 'debian', 'control')])

        (code, out, err) = self.runtest([p])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should build and install package
        self.assertIn('dh build', err)
        self.assertRegex(out, r'binaries.*testpkg')
        self.assertRegex(out, r'Unpacking.*aspell-doc')

        # should show test stdout
        self.assertRegex(out, r'(^|\n)GOOD\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_dsc_build_needed_success(self):
        '''dsc, build-needed restriction, test success'''

        p = self.build_dsc('Tests: pass\nDepends:\nRestrictions: build-needed\n',
                           {'pass': '#!/bin/sh -e\n./test_built | grep -q "built script OK"; '
                            'echo XXX $USER $(whoami) YYY'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        # test should succeed
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should build package
        self.assertIn('dh build', err)

        # test should run as user
        lines = [line for line in out.splitlines() if line.startswith('XXX')]
        self.assertEqual(len(lines), 1, lines)
        fields = lines[0].split()
        self.assertEqual(len(fields), 4)
        self.assertNotEqual(fields[1], 'root')
        self.assertEqual(fields[1], fields[2])
        self.assertEqual(fields[3], 'YYY')

    def test_setup_commands(self):
        '''--setup-commands'''

        p = self.build_src('Tests: t1 t2\nDepends:\n',
                           {'t1': '#!/bin/sh -e\ncat /setup.log /setup_boot.log; echo t1ok\n',
                            't2': '#!/bin/sh -e\ncat /setup.log /setup_boot.log; echo t2ok\n'})

        # we expect a normal user for LXC
        (code, out, err) = self.runtest(['-B', '-d', p, '--setup-commands',
                                         '[ -n "$AUTOPKGTEST_NORMAL_USER" ]; getent passwd $AUTOPKGTEST_NORMAL_USER;'
                                         'echo setupok >> /setup.log',
                                         '--setup-commands-boot', 'echo bootok >> /setup_boot.log'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r't1\s+PASS', out)
        self.assertRegex(out, r't2\s+PASS', out)

        # should show test stdout
        self.assertRegex(out, r'^\w+:x:.*:.*:')
        self.assertIn('\nsetupok\nbootok\nt1ok\n', out)
        self.assertIn('\nsetupok\nbootok\nt2ok\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not have rebooted for these commands
        self.assertIn('@@@@@@ test bed setup', err)
        self.assertNotIn('rebooting testbed', err)

    def test_setup_commands_suppress_reboot(self):
        '''--setup-commands with suppressed reboot'''

        p = self.build_src('Tests: t\nDepends:\n',
                           {'t': '#!/bin/sh -e\ntest -e /run/autopkgtest_no_reboot.stamp'})

        (code, out, err) = self.runtest(['-B', '-d', p, '--setup-commands',
                                         'apt-get install --reinstall --no-install-recommends -y cron; touch /run/autopkgtest_no_reboot.stamp'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r't\s+PASS', out)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should not have rebooted
        self.assertIn('@@@@@@ test bed setup', err)
        self.assertNotIn('rebooting testbed', err)

    def test_apt_source(self):
        '''apt source'''

        (code, out, err) = self.runtest(['-d', 'gdk-pixbuf'])
        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'Unpacking libgdk-pixbuf.*-dev')
        self.assertRegex(out, r'build\s+PASS', out)

        # should show test stdout
        self.assertIn('OK\n', out)
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_user(self):
        '''Run tests as different user'''

        p = self.build_src('Tests: r\nDepends:\nRestrictions: rw-build-tree, needs-root\n\n'
                           'Tests: t\nDepends: aspell-doc\nRestrictions: rw-build-tree\n',
                           {'r': '''#!/bin/sh -e
                                    mkdir xsrc
                                    echo r00t > xsrc/root.txt
                                    ''',
                            't': '''#!/bin/sh -e
                                    echo world > $AUTOPKGTEST_TMP/hello.txt
                                    cat $AUTOPKGTEST_TMP/hello.txt
                                    mkdir -p $AUTOPKGTEST_TMP/one/subdir
                                    echo deep > $AUTOPKGTEST_TMP/one/subdir/subdir.txt
                                    cat $AUTOPKGTEST_TMP/one/subdir/subdir.txt
                                    rm -r $AUTOPKGTEST_TMP/one
                                    echo dir_is_rw > rw_flag
                                    echo user >> xsrc/root.txt
                                    cat rw_flag
                                    echo hello > $AUTOPKGTEST_ARTIFACTS/world.txt
                                    chmod 600 $AUTOPKGTEST_ARTIFACTS/world.txt
                                    echo xxtmpdir > ${TMPDIR:=/tmp}/xxtmpdir.txt
                                    cat $TMPDIR/xxtmpdir.txt
                                    whoami'''})

        outdir = os.path.join(self.workdir, 'out')
        (code, out, err) = self.runtest(['-B', p, '--user=nobody',
                                         '--output-dir', outdir])
        self.assertEqual(code, 0, err)

        # test should succeed
        self.assertRegex(out, r't\s+PASS', out)

        # has output from cats and whoami
        self.assertIn('world\ndeep\ndir_is_rw\nxxtmpdir\nnobody\n', out)

        # should install dependencies
        self.assertIn('Unpacking aspell-doc', out)

        # can write artifacts
        with open(os.path.join(outdir, 'artifacts', 'world.txt')) as f:
            self.assertEqual(f.read(), 'hello\n')

    def test_user_needs_root(self):
        '''--user option with needs-root restriction'''

        p = self.build_src('Tests: t\nDepends: aspell-doc\nRestrictions: needs-root',
                           {'t': '''#!/bin/sh -e
                                    echo world > ${TMPDIR:=/tmp}/hello.txt
                                    cat $TMPDIR/hello.txt
                                    rm $TMPDIR/hello.txt
                                    whoami
                                    echo "USER: $USER"
                                 '''})

        (code, out, err) = self.runtest(['--no-built-binaries', p, '--user=nobody'])
        self.assertEqual(code, 0, err)

        # should install dependencies
        self.assertIn('Unpacking aspell-doc', out)

        # test should succeed
        self.assertRegex(out, r't\s+PASS', out)

        # has output from cat and whoami
        self.assertRegex(out, r'(^|\n)world\nroot\nUSER: root\n')

    def test_copy_timeout(self):
        '''handling copying timeout'''

        p = self.build_src('Tests: to\nDepends:\n',
                           {'to': '#!/bin/sh\ntruncate -s 10G $AUTOPKGTEST_ARTIFACTS/huge\n'})

        outdir = os.path.join(self.workdir, 'out')
        (code, out, err) = self.runtest(['--output-dir=' + outdir,
                                         '--timeout-copy=1',
                                         '-B', p])

        # test should time out
        self.assertEqual(code, 16, err)
        self.assertRegex(out, r'to\s+PASS', out)

        # should have copied parts of the file
        huge_size = os.path.getsize(os.path.join(outdir, 'artifacts', 'huge'))
        self.assertGreater(huge_size, 100000)
        self.assertLess(huge_size, 10000000000)

        # we don't want the raw exception
        self.assertNotIn("Timeout", err)
        self.assertNotIn("Traceback", err)
        # but a proper error message, with cleanup handlers
        self.assertIn("got `timeout', expected `ok...'", err)

    def test_copy_performance(self):
        '''copying files between host and testbed is fast and reliable'''

        p = self.build_src('Tests: t\nDepends:\n',
                           {'t': '#!/bin/sh\ncp -r . $AUTOPKGTEST_ARTIFACTS/tree\n'})
        # create ~ 100 MB of fake source tree (~ 1000 files with 100 kB each)
        block = b'0123456789' * 10000
        for dirnum in range(33):
            d = os.path.join(p, str(dirnum))
            os.mkdir(d)
            for filenum in range(33):
                with open(os.path.join(d, 'src%i' % filenum), 'wb') as f:
                    f.write(block)

        # we expect at least 20 MB/s, thus give it 6 seconds
        outdir = os.path.join(self.workdir, 'out')
        (code, out, err) = self.runtest(['--output-dir=' + outdir,
                                         '--timeout-copy=6',
                                         '-B', p])

        # test should succeed
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r't\s+PASS', out)

        # check integrity of the copy
        subprocess.check_call(['diff', '-Nur', p,
                               os.path.join(outdir, 'artifacts', 'tree')])


class DebTestsVirtContainer(DebTestsVirtFS):
    '''Common tests for runners with container virtualization'''

    def test_logind(self):
        '''test has logind session'''

        p = self.build_src('Tests: logind\nDepends:\n',
                           {'logind': '#!/bin/sh -e\n[ -n "$XDG_SESSION_ID" ] || { echo "no XDG_SESSION_ID" >&2; exit 1; }; '
                            'O=$(loginctl show-session $XDG_SESSION_ID);'
                            'echo "$O" | grep -q "^Active=yes" || echo "$O" >&2; '
                            'echo "$O" | grep -q "^Name=$USER" || echo "$O" >&2; '})

        # testbed might not have libpam-systemd
        (code, out, err) = self.runtest(['--setup-commands', 'apt-get install -y libpam-systemd',
                                         '--no-built-binaries', p])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'logind\s+PASS', out)

    def test_no_locale_conf(self):
        '''No /etc/default/locale'''

        p = self.build_src('Test-Command: locale; echo TestOK\nDepends:\n', {})

        # we expect a normal user for LXC
        (code, out, err) = self.runtest(['-B', p, '--setup-commands',
                                         'rm -f /etc/default/locale; sed -i "/^L/d" /etc/environment'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'command1\s+PASS')
        self.assertRegex(out, r'LC_MESSAGES="C.UTF-8"')
        self.assertRegex(out, r'TestOK')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_timeout(self):
        '''handling test timeout'''

        p = self.build_dsc('Tests: totest\nDepends:\n',
                           {'totest': '#!/bin/sh\necho start\nsleep 10\necho after_sleep\n'})

        (code, out, err) = self.runtest(['-B', '-d', '--timeout-test=3', p])
        # test should time out
        self.assertEqual(code, 4, err)
        self.assertIn("timed out on command", err)
        self.assertRegex(out, r'totest\s+FAIL timed out', out)

        # should show test stdout
        self.assertIn('start', out)
        # but not let the test finish
        self.assertNotIn('after_sleep', out)

    def test_reboot(self):
        '''test that reboots'''

        p = self.build_src('Tests: s r\nDepends:\nRestrictions: needs-root, needs-reboot',
                           {'s': '#!/bin/sh\necho simple_done',
                            'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          mark1) cat /run/bootstamp; echo "test in mark1"; echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt; /tmp/autopkgtest-reboot mark2 ;;
          mark2) cat /run/bootstamp; echo "test in mark2"; echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt ;;
          "") echo "test beginning"; echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt; /tmp/autopkgtest-reboot mark1 ;;
          *) echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2; exit 1 ;;
        esac
        echo "test end"'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['--output-dir=' + outdir, '-B', p,
                                         '--setup-commands-boot=echo BOOTSTAMP >/run/bootstamp'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r's\s+PASS', out)
        self.assertRegex(out, r'r\s+PASS', out)

        # should have all three phases
        self.assertIn('simple_done\n', out)
        self.assertIn('\ntest beginning\nBOOTSTAMP\ntest in mark1\nBOOTSTAMP\ntest in mark2\ntest end\n', out)
        self.assertIn('test process requested reboot with marker mark1', err)
        self.assertIn('test process requested reboot with marker mark2', err)

        self.assertRegex(err, r'testbed running kernel: Linux \d')
        self.assertNotIn('running kernel changed', err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

    def test_reboot_prepare(self):
        '''test that reboots by itself'''

        p = self.build_src('Tests: s r\nDepends:\nRestrictions: needs-root, needs-reboot',
                           {'s': '#!/bin/sh\necho simple_done',
                            'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          "")
              echo "test beginning"
              echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt
              /tmp/autopkgtest-reboot-prepare mark1
              touch /var/post-prepare-1
              reboot ;;
          mark1)
              echo "test in mark1"
              echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt
              /tmp/autopkgtest-reboot-prepare mark2
              touch /var/post-prepare-2
              printf '#!/bin/sh\necho Linux 1.2.3-testy foobar' > /usr/local/bin/uname
              chmod 755 /usr/local/bin/uname
              reboot ;;
          mark2)
              echo "test in mark2"
              echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt
              ls /var/post-prepare-1 /var/post-prepare-2 ;;
          *)
              echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2
              exit 1 ;;
        esac'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['--output-dir=' + outdir, '-B', p + '//'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r's\s+PASS', out)
        self.assertRegex(out, r'r\s+PASS', out)

        # should have all three phases
        self.assertIn('simple_done\n', out)
        self.assertIn('\ntest beginning\ntest in mark1\n'
                      'test in mark2\n/var/post-prepare-1\n/var/post-prepare-2\n', out)
        self.assertIn('test process requested preparation for reboot with marker mark1', err)
        self.assertIn('test process requested preparation for reboot with marker mark2', err)

        self.assertIn('running kernel changed: Linux 1.2.3-testy foobar (current test: r, last reboot marker: mark2)\n', err)
        self.assertNotIn('running kernel changed: Linux 1.2.3-testy foobar (current test s', err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # check artifacts
        with open(os.path.join(outdir, 'artifacts', 'begin.txt')) as f:
            self.assertEqual(f.read(), 'zero\n')
        with open(os.path.join(outdir, 'artifacts', 'mark1.txt')) as f:
            self.assertEqual(f.read(), 'one\n')
        with open(os.path.join(outdir, 'artifacts', 'mark2.txt')) as f:
            self.assertEqual(f.read(), 'two\n')
        with open(os.path.join(outdir, 'testinfo.json')) as f:
            info = json.load(f)
        self.assertEqual(info['test_kernel_versions'],
                         [['r', 'mark2', 'Linux 1.2.3-testy foobar']])
        # this check only works in LXC where the host kernel is used, not in Qemu
        if 'Qemu' not in self.__class__.__name__:
            u = os.uname()
            self.assertEqual(info['kernel_version'],
                             '%s %s %s' % (u.sysname, u.release, u.version.strip()))

    def test_exit_255(self):
        '''test exits with code 255

        This should not be considered a failure of the auxverb.
        '''
        p = self.build_src('Tests: t255\nDepends:\n', {'t255': '#!/bin/sh\nexit 255'})

        (code, out, err) = self.runtest(['-B', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r't255\s+FAIL non-zero exit status 253')
        self.assertNotIn('testbed auxverb failed with', err)

    def test_background_process(self):
        '''leftover background process in test'''

        # leak sleep processes during build and test
        p = self.build_src('Tests: bg\nDepends:\nRestrictions: build-needed',
                           {'bg': '#!/bin/bash -e\nexec -a bg_daemon sleep 30 &\n'
                                  'echo TEST DONE\n'})

        with open(os.path.join(p, 'debian', 'rules'), 'a') as f:
            f.write('\n\noverride_dh_auto_build:\n\tsleep 60 &\n'
                    '\tdh_auto_build\n')

        (code, out, err) = self.runtest(
            ['-d', '-B', p, '--timeout-build=10', '--timeout-test=5'])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'bg\s+PASS', out)
        self.assertIn('TEST DONE\n', out)

    def test_setup_commands_reboot(self):
        '''--setup-commands with reboot'''

        p = self.build_src('Tests: t\nDepends:\n\nTests: r\nDepends:\nRestrictions: needs-root, build-needed\n',
                           {'t': '#!/bin/sh -e\n[ -d "$HOME" ] || echo "HOME not set" >&2\n',
                            'r': '#!/bin/sh -e\n[ -d "$HOME" ] || echo "HOME not set" >&2\n'
                                 '[ "$USER" = "root" ] || echo "USER invalid: $USER" >&2\n'
                                 './test_built\n./test_abspath\n'})

        (code, out, err) = self.runtest(['-B', '-d', p, '--setup-commands',
                                         'apt-get install --reinstall -y cron'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r't\s+PASS', out)
        self.assertRegex(out, r'r\s+PASS', out)

        # built sources should succeed
        self.assertIn('built script OK\nbuilt script OK\n', out)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should have rebooted for these commands
        self.assertIn('@@@@@@ test bed setup', err)
        self.assertIn('rebooting testbed', err)

    def test_apt_source_arch_conflict(self):
        '''apt-source for src with binary that other src also provides for different arch'''

        ar = testarchive.Archive(series='testy', component='main')
        ar.add_sources('testpkg', {'dummy': 'amd64', 'binpkg': 'arm64'}, version='1.0')
        ar.create_deb('dummy', architecture='amd64', srcpkg='testpkg', version='1.0')
        ar.create_deb('binpkg', architecture='arm64', srcpkg='testpkg', version='1.0')
        ar.add_sources('testpkg-arm', {'binpkg': 'amd64'}, version='4.0')
        ar.create_deb('binpkg', architecture='amd64', srcpkg='testpkg-arm', version='4.0')

        apt_sources = os.path.join(self.workdir, 'sources.list')
        with open(apt_sources, 'w') as f:
            f.write('deb [trusted=yes] file:///tmp/testarchive/ testy main\n')
            f.write('deb-src [trusted=yes] file:///tmp/testarchive/ testy main\n')

        (code, out, err) = self.runtest(['--copy', '%s:/tmp/testarchive' % ar.path,
                                         '--copy', '%s:/etc/apt/sources.list' % apt_sources,
                                         '--apt-upgrade',
                                         '--debug', 'testpkg'])
        # this is what we don't want to see; could not figure out correct testpkg version
        self.assertNotIn("Can not find version '4.0' of package 'testpkg'", err)
        # this is what we expect; figured out to use testpkg version 1.0
        self.assertRegex(err, r"apt-get source -d.*testpkg=1\.0")


@unittest.skipUnless('AUTOPKGTEST_TEST_LXC' in os.environ,
                     'Set $AUTOPKGTEST_TEST_LXC to an existing container')
class LxcRunner(AdtTestCase, DebTestsVirtContainer):
    def __init__(self, *args, **kwargs):
        self.container_name = os.environ.get('AUTOPKGTEST_TEST_LXC')
        if self.container_name:
            vargs = ['lxc', self.container_name]
            # check for unprivileged containers
            if os.getuid() > 0 and subprocess.call(
                    ['lxc-info', '-n', self.container_name],
                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT) != 0:
                vargs.insert(1, '--sudo')
        else:
            vargs = []  # tests will be skipped anyway, but we still must be instantiable

        super().__init__(vargs, *args, **kwargs)


@unittest.skipUnless('AUTOPKGTEST_TEST_LXD' in os.environ,
                     'Set $AUTOPKGTEST_TEST_LXD to an existing LXD image')
class LxdRunner(AdtTestCase, DebTestsVirtContainer):
    def __init__(self, *args, **kwargs):
        vargs = ['lxd', os.environ.get('AUTOPKGTEST_TEST_LXD')]
        super().__init__(vargs, *args, **kwargs)


@unittest.skipUnless('AUTOPKGTEST_TEST_QEMU' in os.environ,
                     'Set $AUTOPKGTEST_TEST_QEMU to an existing autopkgtest QEMU image')
class QemuRunner(AdtTestCase, DebTestsVirtContainer):
    def __init__(self, *args, **kwargs):
        super(QemuRunner, self).__init__(['qemu', os.environ.get('AUTOPKGTEST_TEST_QEMU')], *args, **kwargs)

    def test_options_and_baseimage(self):
        '''autopkgtest-virt-qemu options and /dev/baseimage'''

        p = self.build_src('Tests: t\nDepends:',
                           {'t': '''echo CPUNUM; nproc
                                 mkdir -p $AUTOPKGTEST_TMP/one/subdir
                                 echo deep > $AUTOPKGTEST_TMP/one/subdir/subdir.txt
                                 cat $AUTOPKGTEST_TMP/one/subdir/subdir.txt
                                 whoami
                                 test -L /dev/baseimage || echo "no /dev/baseimage link"
                                 blkid -p /dev/baseimage'''})

        (code, out, err) = self.runtest(['-B', '-d', p],
                                        self.virt_args + ['--cpus=2', '--show-boot', '--user=nobody', '--baseimage'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r't\s+PASS', out)

        # expected number of CPUs, cat, and expected user
        self.assertIn('CPUNUM\n2\ndeep\nnobody\n', out)

        # should show boot console output
        self.assertIn(' login:', err)

    def test_reboot_ro(self):
        '''test that reboots, r/o system'''

        # use obsolete $ADT_REBOOT_MARK here to verify backwards compat
        p = self.build_src('Tests: s r\nDepends:\nRestrictions: needs-root breaks-testbed',
                           {'s': '#!/bin/sh\necho simple_done',
                            'r': '''#!/bin/sh -e
        case "$ADT_REBOOT_MARK" in
          mark1) echo "test in mark1"; echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt; /tmp/autopkgtest-reboot mark2 ;;
          mark2) echo "test in mark2"; echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt ;;
          "") echo "test beginning"; echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt; /tmp/autopkgtest-reboot mark1 ;;
          *) echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2; exit 1 ;;
        esac
        echo "test end"'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        ro_sbin = 'M=$(mktemp --directory /run/ro-sbin.XXXXX); ' \
                  'mount -t tmpfs tmpfs $M; ' \
                  'cp -a /sbin $M; cp -a /var/cache $M; ' \
                  'mount -o remount,ro $M; ' \
                  'mount -o bind,ro $M/sbin /sbin; ' \
                  'mount -o bind,ro $M/cache /var/cache; '

        (code, out, err) = self.runtest(['--output-dir=' + outdir, '-B', p,
                                         '--setup-commands', ro_sbin])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r's\s+PASS', out)
        self.assertRegex(out, r'r\s+PASS', out)

        # should have all three phases
        self.assertIn('simple_done\n', out)
        self.assertIn('\ntest beginning\ntest in mark1\ntest in mark2\ntest end\n', out)
        self.assertIn('test process requested reboot with marker mark1', err)
        self.assertIn('test process requested reboot with marker mark2', err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # check artifacts
        with open(os.path.join(outdir, 'artifacts', 'begin.txt')) as f:
            self.assertEqual(f.read(), 'zero\n')
        with open(os.path.join(outdir, 'artifacts', 'mark1.txt')) as f:
            self.assertEqual(f.read(), 'one\n')
        with open(os.path.join(outdir, 'artifacts', 'mark2.txt')) as f:
            self.assertEqual(f.read(), 'two\n')

        # check outdir test stdout/err
        with open(os.path.join(outdir, 'r-stdout')) as f:
            self.assertEqual(f.read(), 'test beginning\ntest in mark1\ntest in mark2\ntest end\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'r-stderr')))

        # should not have any errors from tar or otherwise
        self.assertNotIn('Traceback', err)
        self.assertNotIn('tar:', err)


@unittest.skipUnless('AUTOPKGTEST_TEST_SCHROOT' in os.environ,
                     'Set $AUTOPKGTEST_TEST_SCHROOT to an existing schroot')
class SchrootRunner(AdtTestCase, DebTestsVirtFS, DebTestsFailureModes):
    def __init__(self, *args, **kwargs):
        super(SchrootRunner, self).__init__(['schroot', os.environ.get('AUTOPKGTEST_TEST_SCHROOT')],
                                            *args, **kwargs)

    def test_binary(self):
        '''binary for test'''

        p = self.build_src('Tests: pass\n',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built'})

        # build the package
        subprocess.check_call(['dpkg-buildpackage', '-b', '-us', '-uc', '-tc'],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              cwd=p)
        deb = os.path.join(os.path.dirname(p), 'testpkg_1_all.deb')
        self.assertTrue(os.path.exists(deb))
        # destroy debian/rules, to ensure it does not try to build the package
        # again
        with open(os.path.join(p, 'debian', 'rules'), 'w') as f:
            f.write('bwhahahahano!')

        (code, out, err) = self.runtest(['-d', p, deb])

        # should install package
        self.assertNotIn('dh build', err)
        self.assertIn('Unpacking testpkg', out)
        self.assertRegex(out, r'binaries.*testpkg')

        # test should succeed
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)built script OK\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # our deb should still be there
        self.assertTrue(os.path.exists(deb))

    def test_changes(self):
        '''source/binary .changes'''

        p = self.build_src('Tests: pass\n',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built'})

        # build the package
        subprocess.check_call(['dpkg-buildpackage', '-us', '-uc'],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              cwd=p)
        changes = glob('%s/testpkg_1_*.changes' % os.path.dirname(p))
        self.assertEqual(len(changes), 1)
        changes = changes[0]
        (code, out, err) = self.runtest([changes])

        # should not rebuild
        self.assertNotIn('dh build', err)

        # should install package
        self.assertIn('Unpacking testpkg', out)
        self.assertRegex(out, r'binaries.*testpkg')

        # test should succeed
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)built script OK\n')
        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # our .changes should still be there
        self.assertTrue(os.path.exists(changes))

    def test_logfile(self):
        '''--log-file option'''

        p = self.build_src('Tests: nz\nDepends: coreutils\nRestrictions: build-needed\n',
                           {'nz': '#!/bin/sh\n./test_built\necho I am sick >&2\nexit 7'})

        logfile = os.path.join(self.workdir, 'test.log')
        (code, out, err) = self.runtest(['-d', '--no-built-binaries', p, '--log-file=' + logfile])
        with open(logfile) as f:
            log = f.read()

        # test should fail
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'nz\s+FAIL non-zero exit status 7')
        self.assertRegex(log, r'nz\s+FAIL non-zero exit status 7')

        self.assertIn('processing dependency coreutils', err)
        self.assertIn('processing dependency coreutils', log)

        # should build package
        self.assertIn('dh build', err)
        self.assertIn('dh build', log)

        # should show test stdout
        self.assertRegex(out, r'(^|\n)built script OK\n')
        self.assertRegex(log, r'(^|\n)built script OK\n')
        # should show test stderr
        self.assertRegex(err, r'stderr [ -]+\nI am sick\n')
        self.assertRegex(log, r'stderr [ -]+\nI am sick\n')

    def test_tree_output_dir(self):
        '''source tree, explicit --output-dir'''

        p = self.build_src('Tests: ok\n\nTests: broken\nDepends: @, aspell-doc',
                           {'ok': '#!/bin/sh\n/usr/bin/test_built',
                            'broken': '#!/bin/sh\necho kaputt >&2'})

        outdir = os.path.join(self.workdir, 'out')

        (code, out, err) = self.runtest([p, '--output-dir=' + outdir])

        # test results
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'ok\s+PASS', out)
        self.assertRegex(out, r'broken\s+FAIL stderr: kaputt', out)

        # should show test stdout and stderr
        self.assertRegex(out, r'(^|\n)built script OK\n')
        self.assertRegex(err, r'stderr [ -]+\nkaputt', err)

        # should build package
        self.assertIn('dh build', err)

        # check outdir test stdout/err
        with open(os.path.join(outdir, 'ok-stdout')) as f:
            self.assertEqual(f.read(), 'built script OK\n')
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'ok-stderr')))
        self.assertFalse(os.path.exists(
            os.path.join(outdir, 'broken-stdout')))
        with open(os.path.join(outdir, 'broken-stderr')) as f:
            self.assertEqual(f.read(), 'kaputt\n')

        # check outdir log
        with open(os.path.join(outdir, 'log')) as f:
            contents = f.read()
        self.assertIn('dh build', contents)
        self.assertRegex(contents, r'broken\s+FAIL stderr: kaputt')

        # check summary
        with open(os.path.join(outdir, 'summary')) as f:
            self.assertRegex(f.read(), r'^ok\s+PASS\nbroken\s+FAIL stderr: kaputt$')

        # check test package version
        with open(os.path.join(outdir, 'testpkg-version')) as f:
            contents = f.read()
        self.assertEqual(contents, 'testpkg 1\n')

        # check recorded package lists
        with open(os.path.join(outdir, 'testbed-packages')) as f:
            contents = f.read()
            lines = contents.splitlines()
            self.assertGreater(len(lines), 10)
            self.assertRegex(lines[0], r'^[0-9a-z.-]+\t[0-9a-z.~-]+')
            self.assertRegex(lines[1], r'^[0-9a-z.-]+\t[0-9a-z.~-]+')
            self.assertIn('bash\t', contents)

        # check recorded package list
        for t in ['ok', 'broken']:
            with open(os.path.join(outdir, '%s-packages' % t)) as f:
                contents = f.read()
                self.assertIn('testpkg\t1\n', contents)
                if t == 'broken':
                    self.assertIn('aspell-doc\t', contents)
                else:
                    self.assertNotIn('aspell-doc\t', contents)
                self.assertNotIn('autopkgtest-satdep', contents)

        # check binaries
        self.assertEqual(os.listdir(os.path.join(outdir, 'binaries')), ['testpkg.deb'])

        # check testinfo
        with open(os.path.join(outdir, 'testinfo.json')) as f:
            info = json.load(f)
        u = os.uname()
        self.assertEqual(info['kernel_version'],
                         '%s %s %s' % (u.sysname, u.release, u.version.strip()))
        self.assertNotIn('test_kernel_versions', info)
        self.assertEqual(info['virt_server'], 'autopkgtest-virt-schroot ' + self.virt_args[1])

        # check for cruft in outdir
        self.assertEqual(set(os.listdir(outdir)),
                         set(['log', 'summary', 'binaries', 'testpkg-version',
                              'testbed-packages', 'testinfo.json',
                              'ok-packages', 'ok-stdout', 'broken-packages',
                              'broken-stderr']))

    def test_needs_recommends(self):
        '''needs-recommends restriction'''

        # check that testpkg's Recommends: apt-doc gets installed
        p = self.build_src('Tests: pass\nRestrictions: needs-recommends',
                           {'pass': '#!/bin/sh -e\n/usr/bin/test_built; '
                            'test -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc not installed!" >&2'})

        (code, out, err) = self.runtest([p])
        self.assertEqual(code, 0, out + err)
        self.assertIn('Unpacking apt-doc', out)
        self.assertIn('built script OK', out)

    def test_needs_recommends_last(self):
        '''needs-recommends restriction in later of two tests'''

        # check that testpkg's Recommends: gets installed
        # even if there's a preceding test with the same dependencies
        # but without needs-recommends
        p = self.build_src('Tests: nr\n\nTests: wr\nRestrictions: needs-recommends',
                           {'nr': '#!/bin/sh -e\n'
                            'test ! -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc unexpectedly installed" >&2',
                            'wr': '#!/bin/sh -e\n'
                            'test -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc not installed!" >&2'})

        (code, out, err) = self.runtest([p])
        self.assertEqual(code, 0, out + err)

    def test_needs_recommends_first(self):
        '''needs-recommends restriction in former of two tests'''

        # check that testpkg's Recommends: isn't installed without
        # needs-recommends even if there's a preceding test with the
        # same dependencies but with needs-recommends
        p = self.build_src('Tests: wr\nRestrictions: needs-recommends\n\nTests: nr\n',
                           {'wr': '#!/bin/sh -e\n'
                            'test -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc not installed!" >&2',
                            'nr': '#!/bin/sh -e\n'
                            'test ! -e /usr/share/doc/apt-doc/copyright ||'
                            'echo "apt-doc unexpectedly installed!" >&2'})

        (code, out, err) = self.runtest([p])
        self.assertEqual(code, 0, out + err)

    def test_apt_autodep8(self):
        '''apt, autodep8'''

        (code, out, err) = self.runtest(['libtest-carp-perl'])
        if not have_autodep8:
            self.assertEqual(code, 8, err)
            self.assertIn('SKIP no tests in this package', out)
            return

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'command1\s+PASS', out)

    @unittest.skipUnless(have_autodep8,
                         'autodep8 not installed')
    def test_apt_no_autocontrol(self):
        '''apt, disable automatic test generation'''

        (code, out, err) = self.runtest(['--no-auto-control', 'libtest-carp-perl'])
        self.assertEqual(code, 8, err)
        self.assertIn('SKIP no tests in this package', out)

    def test_broken_test_deps(self):
        '''unsatisfiable test dependencies'''

        p = self.build_src('Tests: p\nDepends: unknown, libc6 (>= 99:99)\n',
                           {'p': '#!/bin/sh -e\ntrue'})

        (code, out, err) = self.runtest(['--no-built-binaries', p])
        self.assertEqual(code, 12, err)
        self.assertRegex(out, r'[dD]epends.*unknown', err)
        self.assertIn('Test dependencies are unsatisfiable', err)
        self.assertRegex(out, r'blame: .*/testpkg', out)
        self.assertIn('Test dependencies are unsatisfiable', out)

    def test_foreign_arch_deps(self):
        '''dependency to foreign architecture'''

        p = self.build_src('Test-Command: objdump -a $(which busybox)\n'
                           'Depends: binutils, busybox-static:i386\n', {})

        (code, out, err) = self.runtest(
            ['--no-built-binaries',
             '--setup-commands', 'dpkg --add-architecture i386 && apt-get update', p])
        self.assertEqual(code, 0)
        self.assertIn('file format elf32-i386', out)

    def test_install_failure(self):
        '''test dependency install failure'''

        p = self.build_src('Tests: p\nDepends: @\n',
                           {'p': '#!/bin/sh -e\ntrue'})
        with open(os.path.join(p, 'debian', 'testpkg.postinst'), 'w') as f:
            f.write('#!/bin/sh -e\necho "BROKEN POSTINST"\nexit 1')

        (code, out, err) = self.runtest([p])
        self.assertEqual(code, 12, err)
        self.assertIn('BROKEN POSTINST', out)
        self.assertRegex(out, r'blame: .*/testpkg', out)
        self.assertIn('badpkg: Test dependencies are unsatisfiable', out)

    def do_apt_pocket_test(self, pocket_args, ver_checks,
                           add_deb_src=False, expect_success=True,
                           extra_control=''):
        '''Common test code for --apt-pocket test'''

        '''--apt-pocket with selected packages from -proposed'''

        ar = testarchive.Archive(series='testy', component='main')
        ar.add_sources('flavor', ['libflavor0', 'mint'], '1')
        ar.create_deb('libflavor0', '1')
        ar.create_deb('vanilla', '1', dependencies={'Depends': 'libflavor0'})
        ar.create_deb('chocolate', '1', dependencies={'Depends': 'libflavor0'})
        ar.create_deb('mint', '1')
        ar.create_deb('honey', '1')

        ap = testarchive.Archive(path=ar.path, pooldir='pool-proposed',
                                 series='testy-proposed', component='main')
        ap.add_sources('flavor', ['libflavor0', 'mint'], '2')
        ap.create_deb('libflavor0', '2')
        ap.create_deb('vanilla', '2', dependencies={'Depends': 'libflavor0 (>= 2)'})
        ap.create_deb('chocolate', '2', dependencies={'Depends': 'libflavor0'})
        ap.create_deb('mint', '2')
        ap.create_deb('honey', '2')

        p = self.build_src('Tests: t\nDepends: vanilla, chocolate, mint, honey\nRestrictions: allow-stderr\n\n' + extra_control,
                           {'t': '''#!/bin/sh -ex\nV=$(dpkg-query --show --showformat='${Version}' vanilla); '''
                                 '''C=$(dpkg-query --show --showformat='${Version}' chocolate); '''
                                 '''M=$(dpkg-query --show --showformat='${Version}' mint); '''
                                 '''H=$(dpkg-query --show --showformat='${Version}' honey); '''
                                 '''L=$(dpkg-query --show --showformat='${Version}' libflavor0); '''
                                 '''apt-cache policy vanilla chocolate mint honey libflavor0; ''' +
                                 ver_checks})

        apt_sources = os.path.join(self.workdir, 'sources.list')
        with open(apt_sources, 'w') as f:
            f.write('deb [trusted=yes] file:///tmp/testarchive/ testy main\n')
            if add_deb_src:
                f.write('deb-src [trusted=yes] file:///tmp/testarchive/ testy main\n')

        (code, out, err) = self.runtest(['--copy', '%s:/tmp/testarchive' % ar.path,
                                         '--copy', '%s:/etc/apt/sources.list' % apt_sources] +
                                        pocket_args +
                                        ['--apt-upgrade',
                                         '--debug', '-B', p])
        if expect_success:
            self.assertEqual(code, 0, err)
        return (code, out, err)

    def test_apt_pocket_all(self):
        '''--apt-pocket with all packages from -proposed'''

        self.do_apt_pocket_test(['--apt-pocket=proposed'],
                                '[ "$V" = 2 ]; [ "$C" = 2 ]; [ "$M" = 2 ]; [ "$H" = 2 ]; [ "$L" = 2 ]; '
                                '[ ! -e /etc/apt/preferences.d/* ]')

    def test_apt_pocket_lib(self):
        '''--apt-pocket selecting library dep from -proposed'''

        self.do_apt_pocket_test(['--apt-pocket=proposed=libflavor0'],
                                '[ "$V" = 1 ]; [ "$C" = 1 ]; [ "$M" = 1 ]; [ "$H" = 1 ]; [ "$L" = 2 ]')

    def test_apt_pocket_selected(self):
        '''--apt-pocket selecting two packages from -proposed'''

        self.do_apt_pocket_test(['--apt-pocket=proposed=chocolate,mint,foo-udeb'],
                                '[ "$V" = 1 ]; [ "$C" = 2 ]; [ "$M" = 2 ]; [ "$H" = 1 ]; [ "$L" = 1 ]')

    def test_apt_pocket_src(self):
        '''--apt-pocket selecting all binaries from a source'''

        self.do_apt_pocket_test(['--apt-pocket=proposed=src:flavor'],
                                '[ "$V" = 1 ]; [ "$C" = 1 ]; [ "$M" = 2 ]; [ "$H" = 1 ]; [ "$L" = 2 ]',
                                add_deb_src=True)

    def test_apt_pocket_pkg_with_proposed_dep(self):
        '''--apt-pocket selecting package from -proposed with dep from -proposed'''

        (code, out, err) = self.do_apt_pocket_test(
            ['--apt-pocket=proposed=vanilla'],
            # FIXME: we really want this, but stopped working with apt 1.1's resolver
            # '[ "$V" = 2 ]; [ "$C" = 1 ]; [ "$M" = 1 ]; [ "$H" = 1 ]; [ "$L" = 2 ]')
            '[ "$V" = 2 ]; [ "$C" = 2 ]; [ "$M" = 2 ]; [ "$H" = 2 ]; [ "$L" = 2 ]')
        # FIXME: our current apt pinning does not resolve this properly, so
        # this will re-try with the entirety of -proposed
        self.assertIn('WARNING: Test dependencies are unsatisfiable with using apt pinning.', err)
        self.assertIn('Retrying with using all packages from testy-proposed\n', err)

    def test_apt_pocket_multi_nonexisting_dep(self):
        '''--apt-pocket with multiple tests and a nonexisting dependency'''

        # include resets of the testbed here to ensure that the setup and
        # error handling work several times in a row in good and bad cases
        (code, out, err) = self.do_apt_pocket_test(
            ['--apt-pocket=proposed=chocolate'],
            '[ "$V" = 1 ]; [ "$C" = 2 ]; [ "$M" = 1 ]; [ "$H" = 1 ]; [ "$L" = 1 ]',
            expect_success=False,
            extra_control='''Test-Command: dpkg-query --show chocolate | grep 2; dpkg-query --show vanilla | grep 1
Depends: vanilla, chocolate

Test-Command: echo "not me" >&2; false
Depends: nonexisting'''
        )
        self.assertEqual(code, 12, err)
        self.assertRegex(out, r't\s+PASS', out)
        self.assertRegex(out, r'command1\s+PASS', out)
        self.assertNotIn(out, 'command2')
        self.assertIn('WARNING: Test dependencies are unsatisfiable with using apt pinning.', err)
        self.assertIn('badpkg: Test dependencies are unsatisfiable', err)

    def test_apt_pocket_no_fallback_fail(self):
        '''--apt-pocket selecting package from -proposed with dep from -proposed with --no-apt-fallback'''

        (code, out, err) = self.do_apt_pocket_test(
            ['--apt-pocket=proposed=vanilla', '--no-apt-fallback'],
            '[ "$V" = 2 ]; [ "$C" = 2 ]; [ "$M" = 2 ]; [ "$H" = 2 ]; [ "$L" = 2 ]',
            expect_success=False)
        self.assertIn('Test dependencies are unsatisfiable.', err)

    def test_apt_pocket_nbs(self):
        '''--apt-pocket with only NBS binaries in unstable'''

        ar = testarchive.Archive(series='testy', component='main')
        ar.add_sources('vanilla', ['vanilla'], '1')
        ar.create_deb('vanilla', '1', dependencies={'Depends': 'libflavor0'})

        ap = testarchive.Archive(path=ar.path, pooldir='pool-proposed',
                                 series='testy-proposed', component='main')
        # version 3 source which is FTBFS (no binaries)
        ap.add_sources('flavor', ['libflavor0', 'mint'], '3')
        # version 2 binaries still stick around
        ap.create_deb('libflavor0', '2', srcpkg='flavor')
        ap.create_deb('mint', '2', srcpkg='flavor')

        apt_sources = os.path.join(self.workdir, 'sources.list')
        with open(apt_sources, 'w') as f:
            f.write('deb [trusted=yes] file:///tmp/testarchive/ testy main\n')
            f.write('deb-src [trusted=yes] file:///tmp/testarchive/ testy main\n')

        (code, out, err) = self.runtest(['--copy', '%s:/tmp/testarchive' % ar.path,
                                         '--copy', '%s:/etc/apt/sources.list' % apt_sources,
                                         '--apt-pocket=proposed',
                                         '--apt-upgrade',
                                         '--debug', 'flavor'])
        self.assertEqual(code, 12, err)

    def test_apt_proposed_dot(self):
        '''--apt-pocket with packages with '.' (regex special character) in name'''

        ar = testarchive.Archive(series='testy', component='main')
        ar.add_sources('flavor', ['libflavor0.0'], '1')
        ar.create_deb('libflavor0.0', '1', srcpkg='flavor')

        ap = testarchive.Archive(path=ar.path, pooldir='pool-proposed',
                                 series='testy-proposed', component='main')
        ap.add_sources('flavor', ['libflavor0.0'], '2')
        ap.create_deb('libflavor0.0', '2', srcpkg='flavor')

        apt_sources = os.path.join(self.workdir, 'sources.list')
        with open(apt_sources, 'w') as f:
            f.write('deb [trusted=yes] file:///tmp/testarchive/ testy main\n')
            f.write('deb-src [trusted=yes] file:///tmp/testarchive/ testy main\n')

        (code, out, err) = self.runtest(['--copy', '%s:/tmp/testarchive' % ar.path,
                                         '--copy', '%s:/etc/apt/sources.list' % apt_sources,
                                         '--apt-pocket=proposed=src:flavor',
                                         '--apt-upgrade',
                                         '--debug', 'flavor'])
        # we cannot actually download the package, but it should get as far as
        # trying, with the right (proposed) version
        self.assertTrue('apt-get source -d -q --only-source flavor=2' in err, err)
        self.assertEqual(code, 12, err)

        return

    def test_apt_pocket_disjoint_binaries(self):
        '''--apt-pocket new version in unstable with non-overlapping binaries'''

        ar = testarchive.Archive(series='testy', component='main')
        ar.add_sources('flavor', ['libflavor0'], '1')
        ar.create_deb('libflavor0', '1', srcpkg='flavor')

        ap = testarchive.Archive(path=ar.path, pooldir='pool-proposed',
                                 series='testy-proposed', component='main')
        ap.add_sources('flavor', ['libflavor2'], '2')
        ap.add_sources('flavor', ['libflavor3'], '3')
        ap.create_deb('libflavor2', '2', srcpkg='flavor')
        ap.create_deb('libflavor3', '3', srcpkg='flavor')

        apt_sources = os.path.join(self.workdir, 'sources.list')
        with open(apt_sources, 'w') as f:
            f.write('deb [trusted=yes] file:///tmp/testarchive/ testy main\n')
            f.write('deb-src [trusted=yes] file:///tmp/testarchive/ testy main\n')

        (code, out, err) = self.runtest(['--copy', '%s:/tmp/testarchive' % ar.path,
                                         '--copy', '%s:/etc/apt/sources.list' % apt_sources,
                                         '--apt-pocket=proposed=src:flavor',
                                         '--apt-upgrade',
                                         '--debug', 'flavor'])
        # we cannot actually download the package, but it should get as far as trying
        self.assertTrue('apt-get source -d -q --only-source flavor=3' in err, err)
        self.assertEqual(code, 12, err)

    def test_git_source_build(self):
        '''Check out and build source from git'''

        (code, out, err) = self.runtest(
            ['--env=DEB_BUILD_OPTIONS=nocheck',
             'https://salsa.debian.org/ci-team/autopkgtest.git'])
        # test should succeed, but skip the Lxd test (if present)
        self.assertIn(code, [0, 2], err)
        self.assertRegex(err, r'autopkgtest\s+PASS')
        self.assertRegex(err, r'installed\s+PASS')
        if code == 2:
            self.assertRegex(err, r'lxd\s+SKIP')

    def test_persistent_apt_failure(self):
        '''persistent apt errors are treated as test failure'''

        self.build_src('Test-Command: true', {})
        (code, out, err) = self.runtest(
            ['--setup-commands', 'echo "deb http://archive.ubuntu.com/ubuntu/ nonexisting main" > /etc/apt/sources.list',
             '--apt-upgrade'])

        self.assertEqual(code, 12, err)
        # apt output should be visible
        self.assertIn('nonexisting/main', err)
        self.assertIn('testbed setup commands failed with status 100\n', err)

    def test_transient_apt_failure(self):
        '''transient apt errors are treated as testbed failure'''

        self.build_src('Test-Command: true', {})
        (code, out, err) = self.runtest(
            ['--setup-commands', 'echo "deb http://nonexist.ing/ nonexisting main" > /etc/apt/sources.list',
             '--apt-upgrade'])

        self.assertEqual(code, 16, err)
        # apt output should be visible
        self.assertIn('nonexisting/main', err)
        self.assertIn('testbed setup commands failed with status 1\n', err)

    def test_tree_apply_patches_unbuilt_tree(self):
        '''source tree, 3.0 (quilt) patches get applied on unbuilt tree'''

        p = self.build_src('Tests: pass\nDepends: coreutils',
                           {'pass': '#!/bin/sh -e\ndebian/testpkg/usr/bin/test_built\n'
                            './test_static\n'})

        # add patch
        patchdir = os.path.join(p, 'debian', 'patches')
        os.mkdir(patchdir)
        with open(os.path.join(patchdir, '01_hack.patch'), 'w') as f:
            f.write('''--- testpkg.orig/test_static
+++ testpkg/test_static
@@ -1,2 +1,2 @@
 #!/bin/sh
-echo "static script OK"
+echo "static patched script OK"
''')
        with open(os.path.join(patchdir, 'series'), 'w') as f:
            f.write('01_hack.patch')

        # turn into 3.0 (quilt) source
        dsrcdir = os.path.join(p, 'debian', 'source')
        os.mkdir(dsrcdir)
        with open(os.path.join(dsrcdir, 'format'), 'w') as f:
            f.write('3.0 (quilt)\n')

        # run tests, should apply unapplied patches
        (code, out, err) = self.runtest(['-d', p])

        # test should succeed
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertEqual(code, 0, err)

        # should have patched source
        self.assertRegex(err, r'dpkg-source:.*01_hack.patch')
        self.assertIn('built script OK\nstatic patched script OK\n', out)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # should build package
        self.assertIn('dh build', err)

    def _build_testprovider(self, provision, arch='all'):
        '''mangle a 'testprovider' package from the testpkg sources'''

        pr = self.build_src('Tests: pass',
                            {},
                            'testprovider')

        subprocess.check_call(['sed', '-i', 's/testpkg/testprovider/g'] +
                              [os.path.join(pr, 'debian', f) for f in ('control', 'changelog', 'links')])

        # fix conflicting files
        subprocess.check_call(['sed', '-i', 's,/usr/bin,/usr/share/testprovider,'] +
                              [os.path.join(pr, 'Makefile')])

        # add a (possibly versioned) Provides
        subprocess.check_call(['sed', '-i', r'/^Depends:/ a\Provides: %s' % provision,
                               os.path.join(pr, 'debian', 'control')])

        if arch != 'all':
            subprocess.check_call(['sed', '-i', 's/Architecture: all/Architecture: %s/' % arch,
                                   os.path.join(pr, 'debian', 'control')])
        subprocess.check_call(['dpkg-buildpackage', '-b', '-us', '-uc', '-tc'],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              cwd=pr)

        shutil.rmtree(pr)

        if arch == 'all':
            deb = os.path.join(os.path.dirname(pr), 'testprovider_1_all.deb')
        else:
            deb = os.path.join(os.path.dirname(pr), 'testprovider_1_' + host_arch + '.deb')
        return deb

    def _test_provides(self, provision, test_dependency, arch='all', control_extra='', expect_failure=False):
        '''internal function to test different combinations of provides support'''

        testprovider_deb = self._build_testprovider(provision, arch)

        # we check that the testprovider package isn't enough to satisfy
        # testpkg's test-dependency on itself via 'test_dependency'
        # even though testprovider Provides: a versioned testpkg
        p = self.build_src('Tests: pass\nDepends: testprovider, %s' % test_dependency,
                           {'pass': '#!/bin/sh -e\ndpkg-query -W -f\\${Status} testpkg | grep -q ^install'})

        if arch != 'all':
            subprocess.check_call(['sed', '-i', 's/Architecture: all/Architecture: %s/' % arch,
                                   os.path.join(p, 'debian', 'control')])

        extradebs = []
        if control_extra:
            with open(os.path.join(p, 'debian', 'control'), 'a') as f:
                f.write('\n' + control_extra)
            extradebs = [os.path.join(os.path.dirname(p), pkg + '_1_all.deb')
                         for pkg in re.findall('^Package: (.+)', control_extra, re.MULTILINE)]

        subprocess.check_call(['dpkg-buildpackage', '-b', '-us', '-uc', '-tc'],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                              cwd=p)

        if arch == 'all':
            testpkg_deb = os.path.join(os.path.dirname(p), 'testpkg_1_all.deb')
        else:
            testpkg_deb = os.path.join(os.path.dirname(p), 'testpkg_1_' + host_arch + '.deb')

        failed = False
        (code, out, err) = self.runtest([p, testpkg_deb, testprovider_deb] + extradebs)
        try:
            self.assertEqual(code, 0, out + err)
            self.assertRegex(out, r'pass\s+PASS', out)
        except BaseException:
            if not expect_failure:
                raise
            failed = True

        shutil.rmtree(p)

        # now check that testpkg gets installed even if testprovider already is
        p = self.build_src('Tests: pass\nDepends: %s' % test_dependency,
                           {'pass': '#!/bin/sh -e\ndpkg-query -W -f\\${Status} testpkg | grep -q ^install'})
        if control_extra:
            with open(os.path.join(p, 'debian', 'control'), 'a') as f:
                f.write('\n' + control_extra)

        tb_deb = '/tmp/provider.deb'

        (code, out, err) = self.runtest(['--copy', '%s:%s' % (testprovider_deb, tb_deb),
                                         '--setup-commands', 'dpkg -i %s' % tb_deb, p])
        try:
            self.assertEqual(code, 0, out + err)
            self.assertRegex(out, r'pass\s+PASS', out)
        except BaseException:
            if not expect_failure:
                raise
            failed = True

        self.assertTrue(failed or not expect_failure)

        shutil.rmtree(p)

    def test_unversioned_provides_with_at(self):
        '''test that unversioned provides do not satisfy test dependencies on "@"'''
        self._test_provides('testpkg', '@')

    def test_unversioned_provides_with_unversioned_explicit(self):
        '''test that unversioned provides do not satisfy unversioned test dependencies on binary package from the same source'''
        self._test_provides('testpkg', 'testpkg')

    # this is somewhat silly but included for completeness
    def test_unversioned_provides_with_versioned_explicit(self):
        '''test that unversioned provides do not satisfy versioned test dependencies on binary package from the same source'''
        self._test_provides('testpkg', 'testpkg (>= 0.5)')

    def test_versioned_provides_with_at(self):
        '''test that versioned provides do not satisfy test dependencies on "@"'''
        self._test_provides('testpkg (= 0.5)', '@')

    def test_versioned_provides_with_unversioned_explicit(self):
        '''test that versioned provides do not satisfy unversioned test dependencies on binary package from the same source'''
        self._test_provides('testpkg (= 0.5)', 'testpkg')

    def test_versioned_provides_with_versioned_explicit(self):
        '''test that versioned provides do not satisfy versioned test dependencies on binary package from the same source'''
        self._test_provides('testpkg (= 0.5)', 'testpkg (>= 0.5)')

    def test_versioned_provides_with_at_arch_any(self):
        '''test that versioned provides do not satisfy test dependencies on "@" with arch:any binary packages'''
        self._test_provides('testpkg (= 0.5)', '@', arch='any')

    def test_versioned_provides_with_at_arch_multiple(self):
        '''test that versioned provides do not satisfy test dependencies on "@" with <os>-any + hurd-any binary packages'''
        self._test_provides('testpkg (= 0.5)', '@', arch=host_os + '-any hurd-any')

    def test_versioned_provides_with_unversioned_explicit_on_not_hurd(self):
        '''test that versioned provides do not satisfy test dependencies on !hurd-any'''
        self._test_provides('testpkg (= 0.5)', 'testpkg [!hurd-any]')

    def test_versioned_provides_with_unversioned_explicit_on_host_arch(self):
        '''test that versioned provides do not satisfy test dependencies on <host_arch>'''
        self._test_provides('testpkg (= 0.5)', 'testpkg [' + host_arch + ']')

    def test_unversioned_provides_with_conflicting_alternatives(self):
        '''test that unversioned provides do not satisfy test dependencies on conflicting alternatives from the same source'''
        testpkg2 = '\nPackage: testpkg2\nArchitecture: all\nDescription: test package 2\nConflicts: testpkg'
        self._test_provides('testpkg', 'testpkg | testpkg2', control_extra=testpkg2)

    def test_versioned_provides_with_conflicting_alternatives(self):
        '''test that versioned provides do not satisfy test dependencies on conflicting alternatives from the same source'''
        testpkg2 = '\nPackage: testpkg2\nArchitecture: all\nDescription: test package 2\nConflicts: testpkg'
        self._test_provides('testpkg (= 0.5)', 'testpkg | testpkg2', control_extra=testpkg2)

    def test_unversioned_provides_with_nonconflicting_alternatives(self):
        '''test that unversioned provides do not satisfy test dependencies on not conflicting alternatives from the same source'''
        testpkg2 = '\nPackage: testpkg2\nArchitecture: all\nDescription: test package 2'
        self._test_provides('testpkg', 'testpkg | testpkg2', control_extra=testpkg2)

    def test_versioned_provides_with_nonconflicting_alternatives(self):
        '''test that versioned provides do not satisfy test dependencies on not conflicting alternatives from the same source'''
        testpkg2 = '\nPackage: testpkg2\nArchitecture: all\nDescription: test package 2'
        self._test_provides('testpkg (= 0.5)', 'testpkg | testpkg2', control_extra=testpkg2)

    def test_unversioned_provides_with_unrelated_alternatives(self):
        '''test that unversioned provides satisfy test dependencies on alternatives not all from the same source'''
        self._test_provides('testpkg', 'testpkg | dpkg', expect_failure=True)

    def test_versioned_provides_with_unrelated_alternatives(self):
        '''test that versioned provides satisfy test dependencies on alternatives not all from the same source'''
        self._test_provides('testpkg (= 0.5)', 'testpkg | dpkg', expect_failure=True)


@unittest.skipUnless('AUTOPKGTEST_TEST_SCHROOT_CLICK' in os.environ,
                     'Set $AUTOPKGTEST_TEST_SCHROOT_CLICK to an existing schroot')
class SchrootClickRunner(AdtTestCase):
    def __init__(self, *args, **kwargs):
        super(SchrootClickRunner, self).__init__(
            ['schroot', os.environ.get('AUTOPKGTEST_TEST_SCHROOT_CLICK')], *args, **kwargs)
        self.click = os.path.join(test_dir, 'testclick_0.1_all.click')
        self.click_src = os.path.join(test_dir, 'testclick')
        self.setup_cmd = 'apt-get install -y --no-install-recommends click ubuntu-sdk-libs; adduser --no-create-home --system clickpkg; chown -R clickpkg:root /opt/click.ubuntu.com'

    def test_click_local_source_implicit(self):
        '''click package with local source, implicit arg type'''

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)
        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '-o', outdir, '--summary-file', sumfile,
             '--setup-commands', self.setup_cmd,
             self.click_src, self.click])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

        # shell test should give expected output
        with open(os.path.join(outdir, 'shell-stdout')) as f:
            self.assertIn('root:x:0', f.read())
        with open(os.path.join(outdir, 'broken-stderr')) as f:
            self.assertEqual(f.read(), 'Bad Things!\n')

        # inst test has additional dependency
        with open(os.path.join(outdir, 'inst-packages')) as f:
            contents = f.read()
        self.assertTrue(contents.startswith('python3-evdev'), contents)

    def test_click_local_source_explicit(self):
        '''click package with local source, explicit arg type'''

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '--summary-file', sumfile,
             '--setup-commands', self.setup_cmd,
             self.click_src, self.click])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

    def test_click_local_source_tmp_install(self):
        '''click package with local source, temp dir deps'''

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '--summary-file', sumfile,
             '--setup-commands', self.setup_cmd,
             '--setup-commands', os.path.join(root_dir, 'setup-commands', 'ro-apt'),
             self.click_src, self.click])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

        self.assertIn('python3-evdev', err)

        # warn about restricted functionality
        self.assertIn('will only work for some packages', err)

    def test_click_preinstalled_all_users(self):
        '''already installed click package for all users, temp dir deps'''

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '--summary-file', sumfile,
             '--copy', '%s:/root/testclick.click' % self.click,
             '--setup-commands', self.setup_cmd,
             '--setup-commands', 'click install --all-users /root/testclick.click',
             '--setup-commands', os.path.join(root_dir, 'setup-commands', 'ro-apt'),
             self.click_src, '--installed-click', 'testclick'])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

    def test_click_preinstalled_user(self):
        '''already installed click package for one user, temp dir deps'''

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['-d', '--summary-file', sumfile,
             '--copy', '%s:/root/testclick.click' % self.click,
             '--setup-commands', self.setup_cmd,
             '--setup-commands', 'click install --user $AUTOPKGTEST_NORMAL_USER /root/testclick.click',
             '--setup-commands', os.path.join(root_dir, 'setup-commands', 'ro-apt'),
             self.click_src, '--installed-click', 'testclick'])
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, '''broken               FAIL stderr: Bad Things!
inst                 PASS
serr                 PASS
shell                PASS
simple               PASS
''')

    def test_override_control(self):
        '''custom manifest with --override-control'''

        custom_manifest = os.path.join(self.workdir, 'myctrl')
        with open(custom_manifest, 'w') as f:
            f.write('''{"name": "testclick",
   "x-test": { "simple": "tests/simple" }
}''')

        sumfile = os.path.join(self.workdir, 'summary')

        (code, out, err) = self.runtest(
            ['--summary-file', sumfile,
             '--setup-commands', self.setup_cmd,
             '--override-control', custom_manifest,
             self.click_src, self.click])
        self.assertEqual(code, 0, err)

        with open(sumfile) as f:
            summary = f.read()
        self.assertEqual(summary, 'simple               PASS\n')


@unittest.skipUnless('AUTOPKGTEST_TEST_LXD' in os.environ,
                     'Set $AUTOPKGTEST_TEST_LXD to an existing container')
class SshRunnerNoScript(AdtTestCase):
    def __init__(self, *args, **kwargs):
        super(SshRunnerNoScript, self).__init__(['invalid'], *args, **kwargs)
        self.info = {}

    def start_container(self, install_key=True, sudo=False, sudo_nopwd=False):
        '''Set up container with SSH'''

        cmd = [os.path.join(test_dir, 'ssh-setup-lxd')]
        if install_key:
            cmd.append('-k')
        if sudo_nopwd:
            cmd.append('-S')
        elif sudo:
            cmd.append('-s')
        cmd += ['open', os.environ.get('AUTOPKGTEST_TEST_LXD')]
        out = subprocess.check_output(cmd, universal_newlines=True)
        for line in out.splitlines():
            (k, v) = line.split('=', 1)
            self.info[k] = v
        self.virt_args = ['ssh', '-d', '-H', self.info['hostname'], '-l', self.info['login'], '-i', self.info['identity']]

    def tearDown(self):
        if self.info:
            subprocess.call([os.path.join(test_dir, 'ssh-setup-lxd'), 'cleanup'] +
                            self.info['extraopts'].split())
            self.info = {}

    def test_no_root(self):
        '''no root'''

        self.start_container()

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr\n\n'
                           'Tests: rootonly\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble\nwhoami',
                            'rootonly': '#!/bin/sh\necho NOTME>&2; exit 1'})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertRegex(out, r'rootonly\s+SKIP.*needs root', out)

        # should show test stdout/err
        self.assertIn('babble\nautopkgtest\n', out)
        self.assertIn('\nI am fine\n', err)
        self.assertRegex(err, r'autopkgtest \[[0-9: -]+\]: test pass: --')

        # no root
        self.assertIn('testbed capabilities: [', err)
        self.assertNotRegex(err, r'testbed capabilities:.*root-on-testbed')

    def test_with_root(self):
        '''with root'''

        self.start_container(sudo=True)

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho hello\nwhoami'})

        (code, out, err) = self.runtest(['-d', '-B', p],
                                        self.virt_args + ['-P', self.info['password']])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'pass\s+PASS', out)

        self.assertIn('hello\nroot\n', out)

        # has root
        self.assertRegex(err, r'testbed capabilities: \[.*root-on-testbed')

    def test_passwordless_sudo(self):
        '''sudo without password'''

        self.start_container(sudo_nopwd=True)

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho hello\nwhoami'})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'pass\s+PASS', out)

        self.assertIn('hello\nroot\n', out)

        # has root
        self.assertRegex(err, r'testbed capabilities: \[.*root-on-testbed')

    @unittest.skip('ssh password auth is not implemented')
    def test_password(self):
        '''no root, password auth'''

        self.start_container(install_key=False)

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble'})

        # drop "-i key" argument
        (code, out, err) = self.runtest(
            ['-d', '-B', p],
            self.virt_args[:-2] + ['-P', self.info['password']])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout/err
        self.assertIn('babble\n', out)
        self.assertRegex(err, r'-+\nI am fine\nautopkgtest \[[0-9: -]+\]: test pass: --')

        # no root
        self.assertIn('testbed capabilities: [', err)
        self.assertNotRegex(err, r'testbed capabilities:.*root-on-testbed')

    def test_background_process(self):
        '''leftover background process in test'''

        self.start_container(sudo_nopwd=True)

        # leak sleep processes during build and test
        p = self.build_src('Tests: bg\nDepends:\nRestrictions: build-needed',
                           {'bg': '#!/bin/bash -e\nexec -a bg_daemon sleep 30 &\n'
                                  'echo TEST DONE\n'})

        with open(os.path.join(p, 'debian', 'rules'), 'a') as f:
            f.write('\n\noverride_dh_auto_build:\n\tsleep 60 &\n'
                    '\tdh_auto_build\n')

        (code, out, err) = self.runtest(['-d', '-B', p, '--timeout-build=10', '--timeout-test=5'])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'bg\s+PASS', out)
        self.assertIn('TEST DONE\n', out)

    def test_reboot(self):
        '''test that reboots'''

        self.start_container(sudo_nopwd=True)
        p = self.build_src('Tests:r\nDepends:\nRestrictions: needs-root',
                           {'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          mark1) echo "test in mark1"; echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt; /tmp/autopkgtest-reboot mark2 ;;
          mark2) echo "test in mark2"; echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt ;;
          "") echo "test beginning"; echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt; /tmp/autopkgtest-reboot mark1 ;;
          *) echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2; exit 1 ;;
        esac
        echo "test end"'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['-d', '-o', outdir, '-B', p],
                                        self.virt_args + ['--reboot'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'r\s+PASS', out)

        # should have all three phases
        self.assertIn('test beginning\ntest in mark1\ntest in mark2\ntest end\n', out)
        self.assertIn('test process requested reboot with marker mark1', err)
        self.assertIn('test process requested reboot with marker mark2', err)

        # check artifacts
        with open(os.path.join(outdir, 'artifacts', 'begin.txt')) as f:
            self.assertEqual(f.read(), 'zero\n')
        with open(os.path.join(outdir, 'artifacts', 'mark1.txt')) as f:
            self.assertEqual(f.read(), 'one\n')
        with open(os.path.join(outdir, 'artifacts', 'mark2.txt')) as f:
            self.assertEqual(f.read(), 'two\n')

    def test_reboot_prepare(self):
        '''test that reboots by itself'''

        self.start_container(sudo_nopwd=True)
        p = self.build_src('Tests: s r\nDepends:\nRestrictions: needs-root',
                           {'s': '#!/bin/sh\necho simple_done',
                            'r': '''#!/bin/sh -e
        case "$AUTOPKGTEST_REBOOT_MARK" in
          "")
              echo "test beginning"
              echo zero > $AUTOPKGTEST_ARTIFACTS/begin.txt
              /tmp/autopkgtest-reboot-prepare mark1
              touch /var/post-prepare-1
              reboot ;;
          mark1)
              echo "test in mark1"
              echo one > $AUTOPKGTEST_ARTIFACTS/mark1.txt
              /tmp/autopkgtest-reboot-prepare mark2
              touch /var/post-prepare-2
              reboot ;;
          mark2)
              echo "test in mark2"
              echo two > $AUTOPKGTEST_ARTIFACTS/mark2.txt
              ls /var/post-prepare-1 /var/post-prepare-2 ;;
          *)
              echo "unknown AUTOPKGTEST_REBOOT_MARK value $AUTOPKGTEST_REBOOT_MARK" >&2
              exit 1 ;;
        esac'''})

        outdir = os.path.join(self.workdir, 'out')
        os.mkdir(outdir)

        (code, out, err) = self.runtest(['-d', '-o', outdir, '-B', p],
                                        self.virt_args + ['--reboot'])

        # test should succeed
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r's\s+PASS', out)
        self.assertRegex(out, r'r\s+PASS', out)

        # should have all three phases
        self.assertIn('simple_done\n', out)
        self.assertIn('\ntest beginning\ntest in mark1\n'
                      'test in mark2\n/var/post-prepare-1\n/var/post-prepare-2\n', out)
        self.assertIn('test process requested preparation for reboot with marker mark1', err)
        self.assertIn('test process requested preparation for reboot with marker mark2', err)

        # should not have any test stderr
        self.assertNotIn(' stderr ', err)

        # check artifacts
        with open(os.path.join(outdir, 'artifacts', 'begin.txt')) as f:
            self.assertEqual(f.read(), 'zero\n')
        with open(os.path.join(outdir, 'artifacts', 'mark1.txt')) as f:
            self.assertEqual(f.read(), 'one\n')
        with open(os.path.join(outdir, 'artifacts', 'mark2.txt')) as f:
            self.assertEqual(f.read(), 'two\n')

    def test_exit_255(self):
        '''test exits with code 255

        This should not be considered a failure of the auxverb.
        '''
        self.start_container()

        p = self.build_src('Tests: t255\nDepends:\n', {'t255': '#!/bin/sh\nexit 255'})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r't255\s+FAIL non-zero exit status 253')
        self.assertNotIn('testbed auxverb failed with', err)

    def test_break_testbed(self):
        '''test breaks sshd'''

        self.start_container()
        p = self.build_src('Test-Command: pkill -e -STOP sshd; sleep 5\nDepends:\n', {})

        (code, out, err) = self.runtest(['-d', '-B', p, '--timeout-test=1', '--timeout-copy=1'])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'command1\s+FAIL timed out')

    def test_isolation(self):
        '''--capability option for isolation'''

        self.start_container()

        p = self.build_src('Tests: mach\nDepends:\nRestrictions: isolation-machine\n\n'
                           'Tests: cont\nDepends:\nRestrictions: isolation-container',
                           {'mach': '#!/bin/sh\nfalse',
                            'cont': '#!/bin/true'})
        (code, out, err) = self.runtest(['-d', '-B', p],
                                        self.virt_args + ['--capability=isolation-container'])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, r'cont\s+PASS', out)
        self.assertRegex(out, r'mach\s+SKIP Test requires machine-level isolation', out)


@unittest.skipUnless('AUTOPKGTEST_TEST_LXD' in os.environ,
                     'Set $AUTOPKGTEST_TEST_LXD to an existing container')
class SshRunnerWithScript(AdtTestCase, DebTestsAll):
    def __init__(self, *args, **kwargs):
        super(SshRunnerWithScript, self).__init__(
            ['ssh', '--debug', '--setup-script', os.path.join(test_dir, 'ssh-setup-lxd'),
             '--', os.environ.get('AUTOPKGTEST_TEST_LXD'), '-k', '-s'],
            *args, **kwargs)

    def test_no_root(self):
        '''no root'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr\n\n'
                           'Tests: rootonly\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble\nwhoami',
                            'rootonly': '#!/bin/sh\necho NOTME>&2; exit 1'})

        # drop -s
        (code, out, err) = self.runtest(['-d', '-B', p],
                                        self.virt_args[:-1])
        self.assertEqual(code, 2, err)
        self.assertRegex(out, r'pass\s+PASS', out)
        self.assertRegex(out, r'rootonly\s+SKIP.*needs root', out)

        # should show test stdout/err
        self.assertIn('babble\nautopkgtest\n', out)
        self.assertIn('\nI am fine\n', err)
        self.assertRegex(err, r'autopkgtest \[[0-9: -]+\]: test pass: --')

        # no root
        self.assertIn('testbed capabilities: [', err)
        self.assertNotRegex(err, r'testbed capabilities:.*root-on-testbed')

    def test_with_root_and_revert(self):
        '''with root and revert'''

        p = self.build_src('Tests: p1\nDepends:\nRestrictions: needs-root, breaks-testbed\n\n'
                           'Tests: p2\nDepends:\n',
                           {'p1': '#!/bin/sh -e\necho hellop1\ntouch /stomp\nwhoami',
                            'p2': '#!/bin/sh -e\n[ ! -e /stomp ]\necho hellop2\nwhoami'})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'p1\s+PASS', out)
        self.assertRegex(out, r'p2\s+PASS', out)

        self.assertIn('hellop1\nroot\n', out)
        self.assertIn('hellop2\nautopkgtest\n', out)

        # check capabilities
        self.assertRegex(err, r'testbed capabilities: \[.*root-on-testbed')
        self.assertRegex(err, r'testbed capabilities: \[.*revert-full-system')

    def test_passwordless_sudo(self):
        '''sudo without password'''

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: needs-root',
                           {'pass': '#!/bin/sh\necho hello\nwhoami'})

        (code, out, err) = self.runtest(['-d', '-B', p],
                                        self.virt_args[:-1] + ['-S'])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'pass\s+PASS', out)

        self.assertIn('hello\nroot\n', out)

        # has root
        self.assertRegex(err, r'testbed capabilities: \[.*root-on-testbed')

    @unittest.skip('ssh password auth is not implemented')
    def test_password(self):
        '''no root, password auth'''

        self.start_container(install_key=False)

        p = self.build_src('Tests: pass\nDepends:\nRestrictions: allow-stderr',
                           {'pass': '#!/bin/sh\necho I am fine >&2\necho babble'})

        # drop -k and -s options
        (code, out, err) = self.runtest(['-d', '-B', p], self.virt_args[:-2])
        self.assertEqual(code, 0, err)
        self.assertRegex(out, r'pass\s+PASS', out)

        # should show test stdout/err
        self.assertIn('babble\n', out)
        self.assertRegex(err, r'-+\nI am fine\nautopkgtest \[[0-9: -]+\]: test pass: --')

        # no root
        self.assertIn('testbed capabilities: [', err)
        self.assertNotRegex(err, r'testbed capabilities:.*root-on-testbed')

    def test_apt_source(self):
        '''apt source'''

        (code, out, err) = self.runtest(['gdk-pixbuf'])
        self.assertEqual(code, 0, out + err)
        self.assertRegex(out, r'build\s+PASS', out)

    def test_click_root(self):
        '''click source, with root'''

        env = os.environ.copy()
        env['AUTOPKGTEST_CLICK_NO_FRAMEWORK_CHECK'] = '1'
        (code, out, err) = self.runtest(
            ['-d',
             '--setup-commands', 'apt-get install -y --no-install-recommends click',
             os.path.join(test_dir, 'testclick'),
             os.path.join(test_dir, 'testclick_0.1_all.click')],
            env=env)
        self.assertEqual(code, 4, err)

        # one test (broken) should fail, the others succeed
        self.assertRegex(out, r'inst\s+PASS')
        self.assertRegex(out, r'serr\s+PASS')
        self.assertRegex(out, r'shell\s+PASS')
        self.assertRegex(out, r'simple\s+PASS')
        self.assertRegex(out, r'broken\s+FAIL stderr: Bad Things!')

        self.assertIn('python3-evdev', err)

    def test_timeout(self):
        '''handling test that times out'''

        p = self.build_src('Tests: to\nDepends:\n',
                           {'to': '#!/bin/sh\necho StartTest; sleep 60; echo NotReached >&2'})

        time_start = time.time()
        (code, out, err) = self.runtest(['-d', '-B', p, '--timeout-test=5'])
        duration = time.time() - time_start
        self.assertEqual(code, 4, err)
        self.assertIn("timed out on command", err)
        self.assertIn('StartTest\n', out)
        self.assertNotIn('PASS', out)
        self.assertRegex(out, r'to\s+FAIL timed out', out)
        self.assertNotIn('NotReached', err)
        self.assertLess(duration, 60, err)

    def test_ssh_failure(self):
        '''ssh failure is testbed failure'''

        p = self.build_src('Tests: f\nDepends:\n', {'f': '#!/bin/sh\npkill sshd'})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 16, err)
        # this is a failure of the auxverb
        self.assertIn('testbed auxverb failed with exit code 255', err)
        self.assertEqual(out.strip(), '')

    def test_command_not_found(self):
        '''command-not-found failure is test failure'''

        p = self.build_src('Tests: f\nDepends:\n', {'f': '#!/bin/sh -e\nno_such_command'})

        (code, out, err) = self.runtest(['-d', '-B', p])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'f\s+FAIL non-zero exit status 127', out)
        # this isn't a failure of the auxverb
        self.assertNotIn('testbed auxverb failed with', err)

    def test_break_testbed(self):
        '''test breaks sshd'''

        p = self.build_src('Test-Command: pkill -e -STOP sshd; sleep 5\nDepends:\n', {})

        (code, out, err) = self.runtest(['-d', '-B', p, '--timeout-test=1', '--timeout-copy=1'])
        self.assertEqual(code, 4, err)
        self.assertRegex(out, r'command1\s+FAIL timed out')


if __name__ == '__main__':
    # Force encoding to UTF-8 even in non-UTF-8 locales.
    import io
    real_stdout = sys.stdout
    assert isinstance(real_stdout, io.TextIOBase)
    sys.stdout = io.TextIOWrapper(real_stdout.detach(), encoding="UTF-8", line_buffering=True)
    unittest.main(testRunner=unittest.TextTestRunner(stream=sys.stdout, verbosity=2))
