diff --git a/charmhelpers/__init__.py b/charmhelpers/__init__.py
index e7aa471541a8a5871df11684ca8c579a30203a80..61ef90719b5d5d759de1a6b80a1ea748d8bb0911 100644
--- a/charmhelpers/__init__.py
+++ b/charmhelpers/__init__.py
@@ -23,22 +23,22 @@ import subprocess
import sys
try:
- import six # flake8: noqa
+ import six # NOQA:F401
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
- import six # flake8: noqa
+ import six # NOQA:F401
try:
- import yaml # flake8: noqa
+ import yaml # NOQA:F401
except ImportError:
if sys.version_info.major == 2:
subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
else:
subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
- import yaml # flake8: noqa
+ import yaml # NOQA:F401
# Holds a list of mapping of mangled function names that have been deprecated
diff --git a/charmhelpers/contrib/hahelpers/apache.py b/charmhelpers/contrib/hahelpers/apache.py
index 605a1becd92a4eb5683d3db28a2267373780a736..2c1e371e179bb6926f227331f86cb14a38c229a2 100644
--- a/charmhelpers/contrib/hahelpers/apache.py
+++ b/charmhelpers/contrib/hahelpers/apache.py
@@ -23,8 +23,8 @@
#
import os
-import subprocess
+from charmhelpers.core import host
from charmhelpers.core.hookenv import (
config as config_get,
relation_get,
@@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file):
def install_ca_cert(ca_cert):
- if ca_cert:
- cert_file = ('/usr/local/share/ca-certificates/'
- 'keystone_juju_ca_cert.crt')
- old_cert = retrieve_ca_cert(cert_file)
- if old_cert and old_cert == ca_cert:
- log("CA cert is the same as installed version", level=INFO)
- else:
- log("Installing new CA cert", level=INFO)
- with open(cert_file, 'wb') as crt:
- crt.write(ca_cert)
- subprocess.check_call(['update-ca-certificates', '--fresh'])
+ host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
diff --git a/charmhelpers/contrib/hardening/apache/checks/config.py b/charmhelpers/contrib/hardening/apache/checks/config.py
index 06482aac1a8efb2bc6ae3e84da355d210287fc88..341da9eee10f73cbe3d7e7e5cf91b57b4d2a89b4 100644
--- a/charmhelpers/contrib/hardening/apache/checks/config.py
+++ b/charmhelpers/contrib/hardening/apache/checks/config.py
@@ -14,6 +14,7 @@
import os
import re
+import six
import subprocess
@@ -95,6 +96,8 @@ class ApacheConfContext(object):
ctxt = settings['hardening']
out = subprocess.check_output(['apache2', '-v'])
+ if six.PY3:
+ out = out.decode('utf-8')
ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
out).group(1)
ctxt['apache_icondir'] = '/usr/share/apache2/icons/'
diff --git a/charmhelpers/contrib/hardening/audits/apache.py b/charmhelpers/contrib/hardening/audits/apache.py
index d32bf44e8fec1afe3cbd0c0d0114a6591ebcbd9c..04825f5ada0c5b0bb9fc0955baa9a10fa199184d 100644
--- a/charmhelpers/contrib/hardening/audits/apache.py
+++ b/charmhelpers/contrib/hardening/audits/apache.py
@@ -15,7 +15,7 @@
import re
import subprocess
-from six import string_types
+import six
from charmhelpers.core.hookenv import (
log,
@@ -35,7 +35,7 @@ class DisabledModuleAudit(BaseAudit):
def __init__(self, modules):
if modules is None:
self.modules = []
- elif isinstance(modules, string_types):
+ elif isinstance(modules, six.string_types):
self.modules = [modules]
else:
self.modules = modules
@@ -69,6 +69,8 @@ class DisabledModuleAudit(BaseAudit):
def _get_loaded_modules():
"""Returns the modules which are enabled in Apache."""
output = subprocess.check_output(['apache2ctl', '-M'])
+ if six.PY3:
+ output = output.decode('utf-8')
modules = []
for line in output.splitlines():
# Each line of the enabled module output looks like:
diff --git a/charmhelpers/contrib/hardening/harden.py b/charmhelpers/contrib/hardening/harden.py
index b55764cdf55e4c94d4adda85e0f3e907c4593a8d..63f21b9c9855065da3be875c01a2c94db7df47b4 100644
--- a/charmhelpers/contrib/hardening/harden.py
+++ b/charmhelpers/contrib/hardening/harden.py
@@ -27,6 +27,8 @@ from charmhelpers.contrib.hardening.ssh.checks import run_ssh_checks
from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks
from charmhelpers.contrib.hardening.apache.checks import run_apache_checks
+_DISABLE_HARDENING_FOR_UNIT_TEST = False
+
def harden(overrides=None):
"""Hardening decorator.
@@ -47,16 +49,28 @@ def harden(overrides=None):
provided with 'harden' config.
:returns: Returns value returned by decorated function once executed.
"""
+ if overrides is None:
+ overrides = []
+
def _harden_inner1(f):
- log("Hardening function '%s'" % (f.__name__), level=DEBUG)
+ # As this has to be py2.7 compat, we can't use nonlocal. Use a trick
+ # to capture the dictionary that can then be updated.
+ _logged = {'done': False}
def _harden_inner2(*args, **kwargs):
+ # knock out hardening via a config var; normally it won't get
+ # disabled.
+ if _DISABLE_HARDENING_FOR_UNIT_TEST:
+ return f(*args, **kwargs)
+ if not _logged['done']:
+ log("Hardening function '%s'" % (f.__name__), level=DEBUG)
+ _logged['done'] = True
RUN_CATALOG = OrderedDict([('os', run_os_checks),
('ssh', run_ssh_checks),
('mysql', run_mysql_checks),
('apache', run_apache_checks)])
- enabled = overrides or (config("harden") or "").split()
+ enabled = overrides[:] or (config("harden") or "").split()
if enabled:
modules_to_run = []
# modules will always be performed in the following order
diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py
index 936b40361bb725415ae9249c0153551033443e47..9133e9b3d25a90a045c81a23934a17e9c5ad4e33 100644
--- a/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/charmhelpers/contrib/openstack/amulet/utils.py
@@ -618,12 +618,12 @@ class OpenStackAmuletUtils(AmuletUtils):
return self.authenticate_keystone(keystone_ip, user, password,
project_name=tenant)
- def authenticate_glance_admin(self, keystone):
+ def authenticate_glance_admin(self, keystone, force_v1_client=False):
"""Authenticates admin user with glance."""
self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
interface='adminURL')
- if keystone.session:
+ if not force_v1_client and keystone.session:
return glance_clientv2.Client("2", session=keystone.session)
else:
return glance_client.Client(ep, token=keystone.auth_token)
@@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils):
nova.flavors.create(name, ram, vcpus, disk, flavorid,
ephemeral, swap, rxtx_factor, is_public)
- def create_cirros_image(self, glance, image_name):
- """Download the latest cirros image and upload it to glance,
- validate and return a resource pointer.
-
- :param glance: pointer to authenticated glance connection
+ def glance_create_image(self, glance, image_name, image_url,
+ download_dir='tests',
+ hypervisor_type=None,
+ disk_format='qcow2',
+ architecture='x86_64',
+ container_format='bare'):
+ """Download an image and upload it to glance, validate its status
+ and return an image object pointer. KVM defaults, can override for
+ LXD.
+
+ :param glance: pointer to authenticated glance api connection
:param image_name: display name for new image
+ :param image_url: url to retrieve
+ :param download_dir: directory to store downloaded image file
+ :param hypervisor_type: glance image hypervisor property
+ :param disk_format: glance image disk format
+ :param architecture: glance image architecture property
+ :param container_format: glance image container format
:returns: glance image pointer
"""
- self.log.debug('Creating glance cirros image '
- '({})...'.format(image_name))
+ self.log.debug('Creating glance image ({}) from '
+ '{}...'.format(image_name, image_url))
- # Download cirros image
+ # Download image
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@@ -700,31 +712,34 @@ class OpenStackAmuletUtils(AmuletUtils):
else:
opener = urllib.FancyURLopener()
- f = opener.open('http://download.cirros-cloud.net/version/released')
- version = f.read().strip()
- cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
- local_path = os.path.join('tests', cirros_img)
-
- if not os.path.exists(local_path):
- cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
- version, cirros_img)
- opener.retrieve(cirros_url, local_path)
- f.close()
+ abs_file_name = os.path.join(download_dir, image_name)
+ if not os.path.exists(abs_file_name):
+ opener.retrieve(image_url, abs_file_name)
# Create glance image
+ glance_properties = {
+ 'architecture': architecture,
+ }
+ if hypervisor_type:
+ glance_properties['hypervisor_type'] = hypervisor_type
+ # Create glance image
if float(glance.version) < 2.0:
- with open(local_path) as fimage:
- image = glance.images.create(name=image_name, is_public=True,
- disk_format='qcow2',
- container_format='bare',
- data=fimage)
+ with open(abs_file_name) as f:
+ image = glance.images.create(
+ name=image_name,
+ is_public=True,
+ disk_format=disk_format,
+ container_format=container_format,
+ properties=glance_properties,
+ data=f)
else:
image = glance.images.create(
name=image_name,
- disk_format="qcow2",
visibility="public",
- container_format="bare")
- glance.images.upload(image.id, open(local_path, 'rb'))
+ disk_format=disk_format,
+ container_format=container_format)
+ glance.images.upload(image.id, open(abs_file_name, 'rb'))
+ glance.images.update(image.id, **glance_properties)
# Wait for image to reach active status
img_id = image.id
@@ -753,15 +768,54 @@ class OpenStackAmuletUtils(AmuletUtils):
val_img_stat, val_img_cfmt, val_img_dfmt))
if val_img_name == image_name and val_img_stat == 'active' \
- and val_img_pub is True and val_img_cfmt == 'bare' \
- and val_img_dfmt == 'qcow2':
+ and val_img_pub is True and val_img_cfmt == container_format \
+ and val_img_dfmt == disk_format:
self.log.debug(msg_attr)
else:
- msg = ('Volume validation failed, {}'.format(msg_attr))
+ msg = ('Image validation failed, {}'.format(msg_attr))
amulet.raise_status(amulet.FAIL, msg=msg)
return image
+ def create_cirros_image(self, glance, image_name, hypervisor_type=None):
+ """Download the latest cirros image and upload it to glance,
+ validate and return a resource pointer.
+
+ :param glance: pointer to authenticated glance connection
+ :param image_name: display name for new image
+ :param hypervisor_type: glance image hypervisor property
+ :returns: glance image pointer
+ """
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'glance_create_image instead of '
+ 'create_cirros_image.')
+
+ self.log.debug('Creating glance cirros image '
+ '({})...'.format(image_name))
+
+ # Get cirros image URL
+ http_proxy = os.getenv('AMULET_HTTP_PROXY')
+ self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
+ if http_proxy:
+ proxies = {'http': http_proxy}
+ opener = urllib.FancyURLopener(proxies)
+ else:
+ opener = urllib.FancyURLopener()
+
+ f = opener.open('http://download.cirros-cloud.net/version/released')
+ version = f.read().strip()
+ cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
+ cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
+ version, cirros_img)
+ f.close()
+
+ return self.glance_create_image(
+ glance,
+ image_name,
+ cirros_url,
+ hypervisor_type=hypervisor_type)
+
def delete_image(self, glance, image):
"""Delete the specified image."""
diff --git a/charmhelpers/contrib/openstack/cert_utils.py b/charmhelpers/contrib/openstack/cert_utils.py
index de853b5371e915fb58bd6b29c1ea16a90ce7d08f..3e078703b124994be2467c167e1df51fd3e350fd 100644
--- a/charmhelpers/contrib/openstack/cert_utils.py
+++ b/charmhelpers/contrib/openstack/cert_utils.py
@@ -25,7 +25,9 @@ from charmhelpers.core.hookenv import (
local_unit,
network_get_primary_address,
config,
+ related_units,
relation_get,
+ relation_ids,
unit_get,
NoNetworkBinding,
log,
@@ -225,3 +227,49 @@ def process_certificates(service_name, relation_id, unit,
create_ip_cert_links(
ssl_dir,
custom_hostname_link=custom_hostname_link)
+
+
+def get_requests_for_local_unit(relation_name=None):
+ """Extract any certificates data targeted at this unit down relation_name.
+
+ :param relation_name: str Name of relation to check for data.
+ :returns: List of bundles of certificates.
+ :rtype: List of dicts
+ """
+ local_name = local_unit().replace('/', '_')
+ raw_certs_key = '{}.processed_requests'.format(local_name)
+ relation_name = relation_name or 'certificates'
+ bundles = []
+ for rid in relation_ids(relation_name):
+ for unit in related_units(rid):
+ data = relation_get(rid=rid, unit=unit)
+ if data.get(raw_certs_key):
+ bundles.append({
+ 'ca': data['ca'],
+ 'chain': data.get('chain'),
+ 'certs': json.loads(data[raw_certs_key])})
+ return bundles
+
+
+def get_bundle_for_cn(cn, relation_name=None):
+ """Extract certificates for the given cn.
+
+ :param cn: str Canonical Name on certificate.
+ :param relation_name: str Relation to check for certificates down.
+ :returns: Dictionary of certificate data,
+ :rtype: dict.
+ """
+ entries = get_requests_for_local_unit(relation_name)
+ cert_bundle = {}
+ for entry in entries:
+ for _cn, bundle in entry['certs'].items():
+ if _cn == cn:
+ cert_bundle = {
+ 'cert': bundle['cert'],
+ 'key': bundle['key'],
+ 'chain': entry['chain'],
+ 'ca': entry['ca']}
+ break
+ if cert_bundle:
+ break
+ return cert_bundle
diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py
index 3e4e82a70f1014bd6f5ed27efc57b89533c849ad..72084cb38e627d4d9ab74c63d1529c212b579820 100644
--- a/charmhelpers/contrib/openstack/context.py
+++ b/charmhelpers/contrib/openstack/context.py
@@ -642,7 +642,7 @@ class HAProxyContext(OSContextGenerator):
return {}
l_unit = local_unit().replace('/', '-')
- cluster_hosts = {}
+ cluster_hosts = collections.OrderedDict()
# NOTE(jamespage): build out map of configured network endpoints
# and associated backends
@@ -1534,10 +1534,15 @@ class NeutronAPIContext(OSContextGenerator):
if 'l2-population' in rdata:
ctxt.update(self.get_neutron_options(rdata))
+ extension_drivers = []
+
if ctxt['enable_qos']:
- ctxt['extension_drivers'] = 'qos'
- else:
- ctxt['extension_drivers'] = ''
+ extension_drivers.append('qos')
+
+ if ctxt['enable_nsg_logging']:
+ extension_drivers.append('log')
+
+ ctxt['extension_drivers'] = ','.join(extension_drivers)
return ctxt
@@ -1897,7 +1902,7 @@ class EnsureDirContext(OSContextGenerator):
Some software requires a user to create a target directory to be
scanned for drop-in files with a specific format. This is why this
context is needed to do that before rendering a template.
- '''
+ '''
def __init__(self, dirname, **kwargs):
'''Used merely to ensure that a given directory exists.'''
@@ -1907,3 +1912,23 @@ class EnsureDirContext(OSContextGenerator):
def __call__(self):
mkdir(self.dirname, **self.kwargs)
return {}
+
+
+class VersionsContext(OSContextGenerator):
+ """Context to return the openstack and operating system versions.
+
+ """
+ def __init__(self, pkg='python-keystone'):
+ """Initialise context.
+
+ :param pkg: Package to extrapolate openstack version from.
+ :type pkg: str
+ """
+ self.pkg = pkg
+
+ def __call__(self):
+ ostack = os_release(self.pkg, base='icehouse')
+ osystem = lsb_release()['DISTRIB_CODENAME'].lower()
+ return {
+ 'openstack_release': ostack,
+ 'operating_system_release': osystem}
diff --git a/charmhelpers/contrib/openstack/ha/utils.py b/charmhelpers/contrib/openstack/ha/utils.py
index 6060ae50b63677126e1941295487cd4460803b10..add8eb9a38b08afbc786b7b1d649dbf40c78a9dd 100644
--- a/charmhelpers/contrib/openstack/ha/utils.py
+++ b/charmhelpers/contrib/openstack/ha/utils.py
@@ -28,6 +28,7 @@ import json
import re
from charmhelpers.core.hookenv import (
+ expected_related_units,
log,
relation_set,
charm_name,
@@ -110,12 +111,17 @@ def assert_charm_supports_dns_ha():
def expect_ha():
""" Determine if the unit expects to be in HA
- Check for VIP or dns-ha settings which indicate the unit should expect to
- be related to hacluster.
+ Check juju goal-state if ha relation is expected, check for VIP or dns-ha
+ settings which indicate the unit should expect to be related to hacluster.
@returns boolean
"""
- return config('vip') or config('dns-ha')
+ ha_related_units = []
+ try:
+ ha_related_units = list(expected_related_units(reltype='ha'))
+ except (NotImplementedError, KeyError):
+ pass
+ return len(ha_related_units) > 0 or config('vip') or config('dns-ha')
def generate_ha_relation_data(service):
diff --git a/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka
index 8e6889e0221e7bbb3db6c8bbb8f5b697e6d40b8d..c281868b16a885cd01a234984974af0349a5d242 100644
--- a/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka
+++ b/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka
@@ -1,12 +1,14 @@
{% if auth_host -%}
[keystone_authtoken]
-auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
-auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
auth_type = password
{% if api_version == "3" -%}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
project_domain_name = {{ admin_domain_name }}
user_domain_name = {{ admin_domain_name }}
{% else -%}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
project_domain_name = default
user_domain_name = default
{% endif -%}
diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py
index ae48d6b43fb43336e1afe28ee3b04fc08f012814..29cad083f8787256bfe3930226ce4c5ef803e90f 100644
--- a/charmhelpers/contrib/openstack/utils.py
+++ b/charmhelpers/contrib/openstack/utils.py
@@ -375,7 +375,7 @@ def get_swift_codename(version):
return codenames[0]
# NOTE: fallback - attempt to match with just major.minor version
- match = re.match('^(\d+)\.(\d+)', version)
+ match = re.match(r'^(\d+)\.(\d+)', version)
if match:
major_minor_version = match.group(0)
for codename, versions in six.iteritems(SWIFT_CODENAMES):
@@ -395,7 +395,7 @@ def get_os_codename_package(package, fatal=True):
out = subprocess.check_output(cmd)
if six.PY3:
out = out.decode('UTF-8')
- except subprocess.CalledProcessError as e:
+ except subprocess.CalledProcessError:
return None
lines = out.split('\n')
for line in lines:
@@ -427,11 +427,11 @@ def get_os_codename_package(package, fatal=True):
vers = apt.upstream_version(pkg.current_ver.ver_str)
if 'swift' in pkg.name:
# Fully x.y.z match for swift versions
- match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
+ match = re.match(r'^(\d+)\.(\d+)\.(\d+)', vers)
else:
# x.y match only for 20XX.X
# and ignore patch level for other packages
- match = re.match('^(\d+)\.(\d+)', vers)
+ match = re.match(r'^(\d+)\.(\d+)', vers)
if match:
vers = match.group(0)
@@ -1450,20 +1450,33 @@ def pausable_restart_on_change(restart_map, stopstart=False,
see core.utils.restart_on_change() for more details.
+ Note restart_map can be a callable, in which case, restart_map is only
+ evaluated at runtime. This means that it is lazy and the underlying
+ function won't be called if the decorated function is never called. Note,
+ retains backwards compatibility for passing a non-callable dictionary.
+
@param f: the function to decorate
- @param restart_map: the restart map {conf_file: [services]}
+ @param restart_map: (optionally callable, which then returns the
+ restart_map) the restart map {conf_file: [services]}
@param stopstart: DEFAULT false; whether to stop, start or just restart
@returns decorator to use a restart_on_change with pausability
"""
def wrap(f):
+ # py27 compatible nonlocal variable. When py3 only, replace with
+ # nonlocal keyword
+ __restart_map_cache = {'cache': None}
+
@functools.wraps(f)
def wrapped_f(*args, **kwargs):
if is_unit_paused_set():
return f(*args, **kwargs)
+ if __restart_map_cache['cache'] is None:
+ __restart_map_cache['cache'] = restart_map() \
+ if callable(restart_map) else restart_map
# otherwise, normal restart_on_change functionality
return restart_on_change_helper(
- (lambda: f(*args, **kwargs)), restart_map, stopstart,
- restart_functions)
+ (lambda: f(*args, **kwargs)), __restart_map_cache['cache'],
+ stopstart, restart_functions)
return wrapped_f
return wrap
diff --git a/charmhelpers/contrib/storage/linux/loopback.py b/charmhelpers/contrib/storage/linux/loopback.py
index 1d6ae6f056bd08660abc0cbc664b326d74d88041..0dfdae523e0a9d4af792031298037af0d58013ce 100644
--- a/charmhelpers/contrib/storage/linux/loopback.py
+++ b/charmhelpers/contrib/storage/linux/loopback.py
@@ -39,7 +39,7 @@ def loopback_devices():
devs = [d.strip().split(' ') for d in
check_output(cmd).splitlines() if d != '']
for dev, _, f in devs:
- loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0]
+ loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0]
return loopbacks
diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py
index 9abf2a45f33ab9725fcdce6dfc3b11232989f422..2e2876596b7490a309f85803b253de6c67ee6f8a 100644
--- a/charmhelpers/core/hookenv.py
+++ b/charmhelpers/core/hookenv.py
@@ -510,6 +510,67 @@ def related_units(relid=None):
subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
+def expected_peer_units():
+ """Get a generator for units we expect to join peer relation based on
+ goal-state.
+
+ The local unit is excluded from the result to make it easy to gauge
+ completion of all peers joining the relation with existing hook tools.
+
+ Example usage:
+ log('peer {} of {} joined peer relation'
+ .format(len(related_units()),
+ len(list(expected_peer_units()))))
+
+ This function will raise NotImplementedError if used with juju versions
+ without goal-state support.
+
+ :returns: iterator
+ :rtype: types.GeneratorType
+ :raises: NotImplementedError
+ """
+ if not has_juju_version("2.4.0"):
+ # goal-state first appeared in 2.4.0.
+ raise NotImplementedError("goal-state")
+ _goal_state = goal_state()
+ return (key for key in _goal_state['units']
+ if '/' in key and key != local_unit())
+
+
+def expected_related_units(reltype=None):
+ """Get a generator for units we expect to join relation based on
+ goal-state.
+
+ Note that you can not use this function for the peer relation, take a look
+ at expected_peer_units() for that.
+
+ This function will raise KeyError if you request information for a
+ relation type for which juju goal-state does not have information. It will
+ raise NotImplementedError if used with juju versions without goal-state
+ support.
+
+ Example usage:
+ log('participant {} of {} joined relation {}'
+ .format(len(related_units()),
+ len(list(expected_related_units())),
+ relation_type()))
+
+ :param reltype: Relation type to list data for, default is to list data for
+ the realtion type we are currently executing a hook for.
+ :type reltype: str
+ :returns: iterator
+ :rtype: types.GeneratorType
+ :raises: KeyError, NotImplementedError
+ """
+ if not has_juju_version("2.4.4"):
+ # goal-state existed in 2.4.0, but did not list individual units to
+ # join a relation in 2.4.1 through 2.4.3. (LP: #1794739)
+ raise NotImplementedError("goal-state relation unit count")
+ reltype = reltype or relation_type()
+ _goal_state = goal_state()
+ return (key for key in _goal_state['relations'][reltype] if '/' in key)
+
+
@cached
def relation_for_unit(unit=None, rid=None):
"""Get the json represenation of a unit's relation"""
@@ -998,6 +1059,7 @@ def application_version_set(version):
@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+@cached
def goal_state():
"""Juju goal state values"""
cmd = ['goal-state', '--format=json']
diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py
index e9fd38a014eba26fc27b2a364cacb3498ad38cce..79953a44dae7870e3191d5c89a6ececd62c00522 100644
--- a/charmhelpers/core/host.py
+++ b/charmhelpers/core/host.py
@@ -34,13 +34,13 @@ import six
from contextlib import contextmanager
from collections import OrderedDict
-from .hookenv import log, DEBUG, local_unit
+from .hookenv import log, INFO, DEBUG, local_unit, charm_name
from .fstab import Fstab
from charmhelpers.osplatform import get_platform
__platform__ = get_platform()
if __platform__ == "ubuntu":
- from charmhelpers.core.host_factory.ubuntu import (
+ from charmhelpers.core.host_factory.ubuntu import ( # NOQA:F401
service_available,
add_new_group,
lsb_release,
@@ -48,7 +48,7 @@ if __platform__ == "ubuntu":
CompareHostReleases,
) # flake8: noqa -- ignore F401 for this import
elif __platform__ == "centos":
- from charmhelpers.core.host_factory.centos import (
+ from charmhelpers.core.host_factory.centos import ( # NOQA:F401
service_available,
add_new_group,
lsb_release,
@@ -58,6 +58,7 @@ elif __platform__ == "centos":
UPDATEDB_PATH = '/etc/updatedb.conf'
+
def service_start(service_name, **kwargs):
"""Start a system service.
@@ -287,8 +288,8 @@ def service_running(service_name, **kwargs):
for key, value in six.iteritems(kwargs):
parameter = '%s=%s' % (key, value)
cmd.append(parameter)
- output = subprocess.check_output(cmd,
- stderr=subprocess.STDOUT).decode('UTF-8')
+ output = subprocess.check_output(
+ cmd, stderr=subprocess.STDOUT).decode('UTF-8')
except subprocess.CalledProcessError:
return False
else:
@@ -442,7 +443,7 @@ def add_user_to_group(username, group):
def chage(username, lastday=None, expiredate=None, inactive=None,
- mindays=None, maxdays=None, root=None, warndays=None):
+ mindays=None, maxdays=None, root=None, warndays=None):
"""Change user password expiry information
:param str username: User to update
@@ -482,8 +483,10 @@ def chage(username, lastday=None, expiredate=None, inactive=None,
cmd.append(username)
subprocess.check_call(cmd)
+
remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
+
def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
"""Replicate the contents of a path"""
options = options or ['--delete', '--executability']
@@ -535,13 +538,15 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
# lets see if we can grab the file and compare the context, to avoid doing
# a write.
existing_content = None
- existing_uid, existing_gid = None, None
+ existing_uid, existing_gid, existing_perms = None, None, None
try:
with open(path, 'rb') as target:
existing_content = target.read()
stat = os.stat(path)
- existing_uid, existing_gid = stat.st_uid, stat.st_gid
- except:
+ existing_uid, existing_gid, existing_perms = (
+ stat.st_uid, stat.st_gid, stat.st_mode
+ )
+ except Exception:
pass
if content != existing_content:
log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
@@ -554,7 +559,7 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
target.write(content)
return
# the contents were the same, but we might still need to change the
- # ownership.
+ # ownership or permissions.
if existing_uid != uid:
log("Changing uid on already existing content: {} -> {}"
.format(existing_uid, uid), level=DEBUG)
@@ -563,6 +568,10 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
log("Changing gid on already existing content: {} -> {}"
.format(existing_gid, gid), level=DEBUG)
os.chown(path, -1, gid)
+ if existing_perms != perms:
+ log("Changing permissions on existing content: {} -> {}"
+ .format(existing_perms, perms), level=DEBUG)
+ os.chmod(path, perms)
def fstab_remove(mp):
@@ -827,7 +836,7 @@ def list_nics(nic_type=None):
ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
ip_output = (line.strip() for line in ip_output if line)
- key = re.compile('^[0-9]+:\s+(.+):')
+ key = re.compile(r'^[0-9]+:\s+(.+):')
for line in ip_output:
matched = re.search(key, line)
if matched:
@@ -1040,3 +1049,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
return modulo * wait
else:
return calculated_wait_time
+
+
+def install_ca_cert(ca_cert, name=None):
+ """
+ Install the given cert as a trusted CA.
+
+ The ``name`` is the stem of the filename where the cert is written, and if
+ not provided, it will default to ``juju-{charm_name}``.
+
+ If the cert is empty or None, or is unchanged, nothing is done.
+ """
+ if not ca_cert:
+ return
+ if not isinstance(ca_cert, bytes):
+ ca_cert = ca_cert.encode('utf8')
+ if not name:
+ name = 'juju-{}'.format(charm_name())
+ cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
+ new_hash = hashlib.md5(ca_cert).hexdigest()
+ if file_hash(cert_file) == new_hash:
+ return
+ log("Installing new CA cert at: {}".format(cert_file), level=INFO)
+ write_file(cert_file, ca_cert)
+ subprocess.check_call(['update-ca-certificates', '--fresh'])
diff --git a/charmhelpers/core/kernel.py b/charmhelpers/core/kernel.py
index 2d404528348e57df4cebee58ff11f6574f334fe0..e01f4f8ba73ee0d5ab7553740c2590a50e42f96d 100644
--- a/charmhelpers/core/kernel.py
+++ b/charmhelpers/core/kernel.py
@@ -26,12 +26,12 @@ from charmhelpers.core.hookenv import (
__platform__ = get_platform()
if __platform__ == "ubuntu":
- from charmhelpers.core.kernel_factory.ubuntu import (
+ from charmhelpers.core.kernel_factory.ubuntu import ( # NOQA:F401
persistent_modprobe,
update_initramfs,
) # flake8: noqa -- ignore F401 for this import
elif __platform__ == "centos":
- from charmhelpers.core.kernel_factory.centos import (
+ from charmhelpers.core.kernel_factory.centos import ( # NOQA:F401
persistent_modprobe,
update_initramfs,
) # flake8: noqa -- ignore F401 for this import
diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py
index ec08cbc27d9c78ceca7322dd07184b43b90b701c..c7ad128c785a805784e3d913e7b45b9dbd98daf0 100644
--- a/charmhelpers/fetch/ubuntu.py
+++ b/charmhelpers/fetch/ubuntu.py
@@ -294,7 +294,7 @@ def apt_unhold(packages, fatal=False):
def import_key(key):
"""Import an ASCII Armor key.
- /!\ A Radix64 format keyid is also supported for backwards
+ A Radix64 format keyid is also supported for backwards
compatibility, but should never be used; the key retrieval
mechanism is insecure and subject to man-in-the-middle attacks
voiding all signature checks using that key.
@@ -454,6 +454,9 @@ def _add_apt_repository(spec):
:param spec: the parameter to pass to add_apt_repository
"""
+ if '{series}' in spec:
+ series = lsb_release()['DISTRIB_CODENAME']
+ spec = spec.replace('{series}', series)
_run_with_retries(['add-apt-repository', '--yes', spec])
diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py
index ed7ea9ecdbb8548e311a703fdc056c9375ce1be6..164d5aa8abbd367a70608526e63fb51d41311aa7 100644
--- a/tests/basic_deployment.py
+++ b/tests/basic_deployment.py
@@ -557,14 +557,6 @@ class CeilometerBasicDeployment(OpenStackAmuletDeployment):
'user_domain_name': user_domain_name,
'username': 'ceilometer',
'password': ks_rel['{}_password'.format(ks_key_prefix)]}
- expected['keystone_authtoken'] = {
- 'auth_uri': auth_uri,
- 'auth_type': 'password',
- 'project_domain_name': project_domain_name,
- 'user_domain_name': user_domain_name,
- 'project_name': 'services',
- 'username': 'ceilometer',
- 'password': ks_rel['{}_password'.format(ks_key_prefix)]}
if self._get_openstack_release() >= self.xenial_ocata:
del expected['api']