import logging
import os
import shutil
import tarfile
from autotest.client.shared import git, error
from autotest.client import utils, os_dep
import data_dir
[docs]class GitRepoParamHelper(git.GitRepoHelper):
'''
Helps to deal with git repos specified in cartersian config files
This class attempts to make it simple to manage a git repo, by using a
naming standard that follows this basic syntax:
<prefix>_name_<suffix>
<prefix> is always 'git_repo' and <suffix> sets options for this git repo.
Example for repo named foo:
git_repo_foo_uri = git://git.foo.org/foo.git
git_repo_foo_base_uri = /home/user/code/foo
git_repo_foo_branch = master
git_repo_foo_lbranch = master
git_repo_foo_commit = bb5fb8e678aabe286e74c4f2993dc2a9e550b627
'''
def __init__(self, params, name, destination_dir):
'''
Instantiates a new GitRepoParamHelper
'''
self.params = params
self.name = name
self.destination_dir = destination_dir
self._parse_params()
def _parse_params(self):
'''
Parses the params items for entries related to this repo
This method currently does everything that the parent class __init__()
method does, that is, sets all instance variables needed by other
methods. That means it's not strictly necessary to call parent's
__init__().
'''
config_prefix = 'git_repo_%s' % self.name
logging.debug('Parsing parameters for git repo %s, configuration '
'prefix is %s' % (self.name, config_prefix))
self.base_uri = self.params.get('%s_base_uri' % config_prefix)
if self.base_uri is None:
logging.debug('Git repo %s base uri is not set' % self.name)
else:
logging.debug('Git repo %s base uri: %s' % (self.name,
self.base_uri))
self.uri = self.params.get('%s_uri' % config_prefix)
logging.debug('Git repo %s uri: %s' % (self.name, self.uri))
self.branch = self.params.get('%s_branch' % config_prefix, 'master')
logging.debug('Git repo %s branch: %s' % (self.name, self.branch))
self.lbranch = self.params.get('%s_lbranch' % config_prefix)
if self.lbranch is None:
self.lbranch = self.branch
logging.debug('Git repo %s lbranch: %s' % (self.name, self.lbranch))
self.commit = self.params.get('%s_commit' % config_prefix)
if self.commit is None:
logging.debug('Git repo %s commit is not set' % self.name)
else:
logging.debug('Git repo %s commit: %s' % (self.name, self.commit))
self.tag = self.params.get('%s_tag' % config_prefix)
if self.tag is None:
logging.debug('Git repo %s tag is not set' % self.name)
else:
logging.debug('Git repo %s tag: %s' % (self.name, self.tag))
self.key_file = None
tag_signed = self.params.get('%s_tag_signed' % config_prefix)
if tag_signed is None:
logging.warning('Git repo %s tag is not signed' % self.name)
logging.warning('This means we will not verify if the key was '
'made by whomever claims to have made it '
'(dangerous)')
else:
self.key_file = os.path.join(data_dir.get_data_dir(), 'gpg',
tag_signed)
if os.path.isfile(self.key_file):
logging.debug('Git repo %s tag %s will be verified with public '
'key file %s', self.name, self.tag, self.key_file)
else:
raise error.TestError('GPG public key file %s not found, will '
'not proceed with testing' %
self.key_file)
self.cmd = os_dep.command('git')
self.recursive = self.params.get('%s_recursive', 'yes')
[docs] def execute(self):
super(GitRepoParamHelper, self).execute()
cwd = os.path.curdir
os.chdir(self.destination_dir)
utils.system('git remote add origin %s' % self.uri, ignore_status=True)
if self.recursive == 'yes':
utils.system('git submodule init')
utils.system('git submodule update')
if self.tag:
utils.system('git checkout %s' % self.tag)
if self.key_file is not None:
try:
gnupg_home = os.path.join(data_dir.get_tmp_dir(),
'gnupg')
if not os.path.isdir(gnupg_home):
os.makedirs(gnupg_home)
os.environ['GNUPGHOME'] = gnupg_home
utils.system('gpg --import %s' % self.key_file)
logging.debug('Verifying if tag is actually signed with '
'GPG key ID %s' % self.key_file)
utils.system('git tag -v %s' % self.tag)
except error.CmdError:
raise error.TestError("GPG signature check for git repo "
"%s failed" % self.name)
# Log the top commit message, good for quick reference
utils.system('git log -1')
os.chdir(cwd)
[docs]class LocalSourceDirHelper(object):
'''
Helper class to deal with source code sitting somewhere in the filesystem
'''
def __init__(self, source_dir, destination_dir):
'''
:param source_dir:
:param destination_dir:
:return: new LocalSourceDirHelper instance
'''
self.source = source_dir
self.destination = destination_dir
[docs] def execute(self):
'''
Copies the source directory to the destination directory
'''
if os.path.isdir(self.destination):
shutil.rmtree(self.destination)
if os.path.isdir(self.source):
shutil.copytree(self.source, self.destination)
[docs]class LocalSourceDirParamHelper(LocalSourceDirHelper):
'''
Helps to deal with source dirs specified in cartersian config files
This class attempts to make it simple to manage a source dir, by using a
naming standard that follows this basic syntax:
<prefix>_name_<suffix>
<prefix> is always 'local_src' and <suffix> sets options for this source
dir. Example for source dir named foo:
local_src_foo_path = /home/user/foo
'''
def __init__(self, params, name, destination_dir):
'''
Instantiate a new LocalSourceDirParamHelper
'''
self.params = params
self.name = name
self.destination_dir = destination_dir
self._parse_params()
def _parse_params(self):
'''
Parses the params items for entries related to source dir
'''
config_prefix = 'local_src_%s' % self.name
logging.debug('Parsing parameters for local source %s, configuration '
'prefix is %s' % (self.name, config_prefix))
self.path = self.params.get('%s_path' % config_prefix)
logging.debug('Local source directory %s path: %s' % (self.name,
self.path))
self.source = self.path
self.destination = self.destination_dir
[docs]class LocalTarHelper(object):
'''
Helper class to deal with source code in a local tarball
'''
def __init__(self, source, destination_dir):
self.source = source
self.destination = destination_dir
[docs] def execute(self):
'''
Executes all action this helper is supposed to perform
This is the main entry point method for this class, and all other
helper classes.
'''
self.extract()
[docs]class LocalTarParamHelper(LocalTarHelper):
'''
Helps to deal with source tarballs specified in cartersian config files
This class attempts to make it simple to manage a tarball with source code,
by using a naming standard that follows this basic syntax:
<prefix>_name_<suffix>
<prefix> is always 'local_tar' and <suffix> sets options for this source
tarball. Example for source tarball named foo:
local_tar_foo_path = /tmp/foo-1.0.tar.gz
'''
def __init__(self, params, name, destination_dir):
'''
Instantiates a new LocalTarParamHelper
'''
self.params = params
self.name = name
self.destination_dir = destination_dir
self._parse_params()
def _parse_params(self):
'''
Parses the params items for entries related to this local tar helper
'''
config_prefix = 'local_tar_%s' % self.name
logging.debug('Parsing parameters for local tar %s, configuration '
'prefix is %s' % (self.name, config_prefix))
self.path = self.params.get('%s_path' % config_prefix)
logging.debug('Local source tar %s path: %s' % (self.name,
self.path))
self.source = self.path
self.destination = self.destination_dir
[docs]class RemoteTarHelper(LocalTarHelper):
'''
Helper that fetches a tarball and extracts it locally
'''
def __init__(self, source_uri, destination_dir):
self.source = source_uri
self.destination = destination_dir
[docs] def execute(self):
'''
Executes all action this helper class is supposed to perform
This is the main entry point method for this class, and all other
helper classes.
This implementation fetches the remote tar file and then extracts
it using the functionality present in the parent class.
'''
name = os.path.basename(self.source)
base_dest = os.path.dirname(self.destination_dir)
dest = os.path.join(base_dest, name)
utils.get_file(self.source, dest)
self.source = dest
self.extract()
[docs]class RemoteTarParamHelper(RemoteTarHelper):
'''
Helps to deal with remote source tarballs specified in cartersian config
This class attempts to make it simple to manage a tarball with source code,
by using a naming standard that follows this basic syntax:
<prefix>_name_<suffix>
<prefix> is always 'local_tar' and <suffix> sets options for this source
tarball. Example for source tarball named foo:
remote_tar_foo_uri = http://foo.org/foo-1.0.tar.gz
'''
def __init__(self, params, name, destination_dir):
'''
Instantiates a new RemoteTarParamHelper instance
'''
self.params = params
self.name = name
self.destination_dir = destination_dir
self._parse_params()
def _parse_params(self):
'''
Parses the params items for entries related to this remote tar helper
'''
config_prefix = 'remote_tar_%s' % self.name
logging.debug('Parsing parameters for remote tar %s, configuration '
'prefix is %s' % (self.name, config_prefix))
self.uri = self.params.get('%s_uri' % config_prefix)
logging.debug('Remote source tar %s uri: %s' % (self.name,
self.uri))
self.source = self.uri
self.destination = self.destination_dir
[docs]class PatchHelper(object):
'''
Helper that encapsulates the patching of source code with patch files
'''
def __init__(self, source_dir, patches):
'''
Initializes a new PatchHelper
'''
self.source_dir = source_dir
self.patches = patches
[docs] def download(self):
'''
Copies patch files from remote locations to the source directory
'''
for patch in self.patches:
utils.get_file(patch, os.path.join(self.source_dir,
os.path.basename(patch)))
[docs] def patch(self):
'''
Patches the source dir with all patch files
'''
os.chdir(self.source_dir)
for patch in self.patches:
utils.system('patch -p1 < %s' % os.path.basename(patch))
[docs] def execute(self):
'''
Performs all steps necessary to download patches and apply them
'''
self.download()
self.patch()
[docs]class PatchParamHelper(PatchHelper):
'''
Helps to deal with patches specified in cartersian config files
This class attempts to make it simple to patch source coude, by using a
naming standard that follows this basic syntax:
[<git_repo>|<local_src>|<local_tar>|<remote_tar>]_<name>_patches
<prefix> is either a 'local_src' or 'git_repo', that, together with <name>
specify a directory containing source code to receive the patches. That is,
for source code coming from git repo foo, patches would be specified as:
git_repo_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
And for for patches to be applied on local source code named also foo:
local_src_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
'''
def __init__(self, params, prefix, source_dir):
'''
Initializes a new PatchParamHelper instance
'''
self.params = params
self.prefix = prefix
self.source_dir = source_dir
self._parse_params()
def _parse_params(self):
'''
Parses the params items for entries related to this set of patches
This method currently does everything that the parent class __init__()
method does, that is, sets all instance variables needed by other
methods. That means it's not strictly necessary to call parent's
__init__().
'''
logging.debug('Parsing patch parameters for prefix %s' % self.prefix)
patches_param_key = '%s_patches' % self.prefix
self.patches_str = self.params.get(patches_param_key, '[]')
logging.debug('Patches config for prefix %s: %s' % (self.prefix,
self.patches_str))
self.patches = eval(self.patches_str)
logging.debug('Patches for prefix %s: %s' % (self.prefix,
", ".join(self.patches)))
[docs]class GnuSourceBuildInvalidSource(Exception):
'''
Exception raised when build source dir/file is not valid
'''
pass
[docs]class SourceBuildFailed(Exception):
'''
Exception raised when building with parallel jobs fails
This serves as feedback for code using
:class:`virttest.build_helper.BuildHelper`.
'''
pass
[docs]class SourceBuildParallelFailed(Exception):
'''
Exception raised when building with parallel jobs fails
This serves as feedback for code using
:class:`virttest.build_helper.BuildHelper`.
'''
pass
[docs]class GnuSourceBuildHelper(object):
'''
Handles software installation of GNU-like source code
This basically means that the build will go though the classic GNU
autotools steps: ./configure, make, make install
'''
def __init__(self, source, build_dir, prefix,
configure_options=[]):
'''
:type source: string
:param source: source directory or tarball
:type prefix: string
:param prefix: installation prefix
:type build_dir: string
:param build_dir: temporary directory used for building the source code
:type configure_options: list
:param configure_options: options to pass to configure
:throws: GnuSourceBuildInvalidSource
'''
self.source = source
self.build_dir = build_dir
self.prefix = prefix
self.configure_options = configure_options
self.install_debug_info = True
self.include_pkg_config_path()
[docs] def include_pkg_config_path(self):
'''
Adds the current prefix to the list of paths that pkg-config searches
This is currently not optional as there is no observed adverse side
effects of enabling this. As the "prefix" is usually only valid during
a test run, we believe that having other pkg-config files (``*.pc``) in
either ``<prefix>/share/pkgconfig`` or ``<prefix>/lib/pkgconfig`` is
exactly for the purpose of using them.
:return: None
'''
env_var = 'PKG_CONFIG_PATH'
include_paths = [os.path.join(self.prefix, 'share', 'pkgconfig'),
os.path.join(self.prefix, 'lib', 'pkgconfig')]
if os.environ.has_key(env_var):
paths = os.environ[env_var].split(':')
for include_path in include_paths:
if include_path not in paths:
paths.append(include_path)
os.environ[env_var] = ':'.join(paths)
else:
os.environ[env_var] = ':'.join(include_paths)
logging.debug('PKG_CONFIG_PATH is: %s' % os.environ['PKG_CONFIG_PATH'])
[docs] def enable_debug_symbols(self):
'''
Enables option that leaves debug symbols on compiled software
This makes debugging a lot easier.
'''
enable_debug_option = "--disable-strip"
if enable_debug_option in self.get_available_configure_options():
self.configure_options.append(enable_debug_option)
logging.debug('Enabling debug symbols with option: %s' %
enable_debug_option)
[docs] def make_parallel(self):
'''
Runs "make" using the correct number of parallel jobs
'''
parallel_make_jobs = utils.count_cpus()
make_command = "make -j %s" % parallel_make_jobs
logging.info("Running parallel make on build dir")
os.chdir(self.build_dir)
utils.system(make_command)
[docs] def make_non_parallel(self):
'''
Runs "make", using a single job
'''
os.chdir(self.build_dir)
utils.system("make")
[docs] def make_clean(self):
'''
Runs "make clean"
'''
os.chdir(self.build_dir)
utils.system("make clean")
[docs] def make(self, failure_feedback=True):
'''
Runs a parallel make, falling back to a single job in failure
:param failure_feedback: return information on build failure by raising
the appropriate exceptions
:raise: SourceBuildParallelFailed if parallel build fails, or
SourceBuildFailed if single job build fails
'''
try:
self.make_parallel()
except error.CmdError:
try:
self.make_clean()
self.make_non_parallel()
except error.CmdError:
if failure_feedback:
raise SourceBuildFailed
if failure_feedback:
raise SourceBuildParallelFailed
[docs] def make_install(self):
'''
Runs "make install"
'''
os.chdir(self.build_dir)
utils.system("make install")
install = make_install
[docs] def execute(self):
'''
Runs appropriate steps for *building* this source code tree
'''
if self.install_debug_info:
self.enable_debug_symbols()
self.configure()
self.make()
[docs]class LinuxKernelBuildHelper(object):
'''
Handles Building Linux Kernel.
'''
def __init__(self, params, prefix, source):
'''
:type params: dict
:param params: dictionary containing the test parameters
:type source: string
:param source: source directory or tarball
:type prefix: string
:param prefix: installation prefix
'''
self.params = params
self.prefix = prefix
self.source = source
self._parse_params()
def _parse_params(self):
'''
Parses the params items for entries related to guest kernel
'''
configure_opt_key = '%s_config' % self.prefix
self.config = self.params.get(configure_opt_key, '')
build_image_key = '%s_build_image' % self.prefix
self.build_image = self.params.get(build_image_key,
'arch/x86/boot/bzImage')
build_target_key = '%s_build_target' % self.prefix
self.build_target = self.params.get(build_target_key, 'bzImage')
kernel_path_key = '%s_kernel_path' % self.prefix
image_dir = os.path.join(data_dir.get_data_dir(), 'images')
default_kernel_path = os.path.join(image_dir,
self.build_target)
self.kernel_path = self.params.get(kernel_path_key,
default_kernel_path)
logging.info('Parsing Linux kernel build parameters for %s',
self.prefix)
[docs] def make_guest_kernel(self):
'''
Runs "make", using a single job
'''
os.chdir(self.source)
logging.info("Building guest kernel")
logging.debug("Kernel config is %s" % self.config)
utils.get_file(self.config, '.config')
# FIXME currently no support for builddir
# run old config
utils.system('yes "" | make oldconfig > /dev/null')
parallel_make_jobs = utils.count_cpus()
make_command = "make -j %s %s" % (
parallel_make_jobs, self.build_target)
logging.info("Running parallel make on src dir")
utils.system(make_command)
[docs] def make_clean(self):
'''
Runs "make clean"
'''
os.chdir(self.source)
utils.system("make clean")
[docs] def make(self, failure_feedback=True):
'''
Runs a parallel make
:param failure_feedback: return information on build failure by raising
the appropriate exceptions
:raise: SourceBuildParallelFailed if parallel build fails, or
'''
try:
self.make_clean()
self.make_guest_kernel()
except error.CmdError:
if failure_feedback:
raise SourceBuildParallelFailed
[docs] def cp_linux_kernel(self):
'''
Copying Linux kernel to target path
'''
os.chdir(self.source)
utils.force_copy(self.build_image, self.kernel_path)
install = cp_linux_kernel
[docs] def execute(self):
'''
Runs appropriate steps for *building* this source code tree
'''
self.make()
[docs]class GnuSourceBuildParamHelper(GnuSourceBuildHelper):
'''
Helps to deal with gnu_autotools build helper in cartersian config files
This class attempts to make it simple to build source coude, by using a
naming standard that follows this basic syntax:
[<git_repo>|<local_src>]_<name>_<option> = value
To pass extra options to the configure script, while building foo from a
git repo, set the following variable:
git_repo_foo_configure_options = --enable-feature
'''
def __init__(self, params, name, destination_dir, install_prefix):
'''
Instantiates a new GnuSourceBuildParamHelper
'''
self.params = params
self.name = name
self.destination_dir = destination_dir
self.install_prefix = install_prefix
self._parse_params()
def _parse_params(self):
'''
Parses the params items for entries related to source directory
This method currently does everything that the parent class __init__()
method does, that is, sets all instance variables needed by other
methods. That means it's not strictly necessary to call parent's
__init__().
'''
logging.debug('Parsing gnu_autotools build parameters for %s' %
self.name)
configure_opt_key = '%s_configure_options' % self.name
configure_options = self.params.get(configure_opt_key, '').split()
logging.debug('Configure options for %s: %s' % (self.name,
configure_options))
self.source = self.destination_dir
self.build_dir = self.destination_dir
self.prefix = self.install_prefix
self.configure_options = configure_options
self.include_pkg_config_path()
# Support the install_debug_info feature, that automatically
# adds/keeps debug information on generated libraries/binaries
install_debug_info_cfg = self.params.get("install_debug_info", "yes")
self.install_debug_info = install_debug_info_cfg != "no"