Merge branch 'stable/19.10' into mergetest

Trying to merge stable/19.10 into master (tag charmRev39)

.coveragerc

-[report]
-# Regexes for lines to exclude from consideration
-exclude_lines =
-    if __name__ == .__main__.:
-include=
-    hooks/keystone_*
-    actions/actions.py

.gitignore

@@ -13,3 +13,4 @@ func-results.json
 .local
 __pycache__
 .stestr
+.idea

.gitreview

 [gerrit]
-host=review.openstack.org
+host=review.opendev.org
 port=29418
 project=openstack/charm-keystone.git
 
-defaultbranch=stable/17.11
+defaultbranch=stable/19.10

.pydevproject

 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <?eclipse-pydev version="1.0"?><pydev_project>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
-<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
-<path>/keystone/hooks</path>
-<path>/keystone/unit_tests</path>
-</pydev_pathproperty>
+    
+    <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+    
+    <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+    
+    <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+        <path>/keystone/hooks</path>
+        <path>/keystone/unit_tests</path>
+        <path>/${PROJECT_DIR_NAME}</path>
+    </pydev_pathproperty>
+    
 </pydev_project>

.stestr.conf

+[DEFAULT]
+test_path=./unit_tests
+top_dir=./

.testr.conf

-[DEFAULT]
-test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
-             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
-             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
-             ${PYTHON:-python} -m subunit.run discover -t ./ ./unit_tests $LISTOPT $IDOPTION
-
-test_id_option=--load-list $IDFILE
-test_list_option=--list

.zuul.yaml

+- project:
+    templates:
+      - python35-charm-jobs
+      - openstack-python3-train-jobs
+      - openstack-cover-jobs

Makefile

@@ -12,8 +12,8 @@ test:
 	@tox -e py27
 
 functional_test:
-	@echo Starting Amulet tests...
-	@tox -e func27
+	@echo Starting Zaza functional tests...
+	@tox -e func
 
 bin/charm_helpers_sync.py:
 	@mkdir -p bin
@@ -22,7 +22,6 @@ bin/charm_helpers_sync.py:
 
 sync: bin/charm_helpers_sync.py
 	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml
-	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml
 
 publish: lint test
 	export OUTPUT=`charm push . cs:~$(USER)/$(NAME)`; echo $$OUTPUT

actions.yaml

@@ -13,3 +13,5 @@ openstack-upgrade:
   description: |
     Perform openstack upgrades. Config option action-managed-upgrade must be
     set to True.
+security-checklist:
+  description: Validate the running configuration against the OpenStack security guides checklist

actions/actions.py

-#!/usr/bin/python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Canonical Ltd
 #
@@ -17,9 +17,21 @@
 import sys
 import os
 
+_path = os.path.dirname(os.path.realpath(__file__))
+_hooks = os.path.abspath(os.path.join(_path, '../hooks'))
+_root = os.path.abspath(os.path.join(_path, '..'))
+
+
+def _add_path(path):
+    if path not in sys.path:
+        sys.path.insert(1, path)
+
+_add_path(_hooks)
+_add_path(_root)
+
 from charmhelpers.core.hookenv import action_fail
 
-from hooks.keystone_utils import (
+from keystone_utils import (
     pause_unit_helper,
     resume_unit_helper,
     register_configs,
@@ -52,7 +64,7 @@ def main(args):
     try:
         action = ACTIONS[action_name]
     except KeyError:
-        return "Action %s undefined" % action_name
+        return "Action {} undefined".format(action_name)
     else:
         try:
             action(args)

actions/charmhelpers

-../charmhelpers
\ No newline at end of file

actions/hooks

-../hooks
\ No newline at end of file

actions/openstack_upgrade.py

-#!/usr/bin/python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Canonical Ltd
 #
@@ -17,7 +17,17 @@
 import os
 import sys
 
-sys.path.append('hooks/')
+_path = os.path.dirname(os.path.realpath(__file__))
+_hooks = os.path.abspath(os.path.join(_path, '../hooks'))
+_root = os.path.abspath(os.path.join(_path, '..'))
+
+
+def _add_path(path):
+    if path not in sys.path:
+        sys.path.insert(1, path)
+
+_add_path(_hooks)
+_add_path(_root)
 
 from charmhelpers.contrib.openstack.utils import (
     do_action_openstack_upgrade,
@@ -43,7 +53,8 @@ def openstack_upgrade():
     if (do_action_openstack_upgrade('keystone',
                                     do_openstack_upgrade,
                                     register_configs())):
-        os.execl('./hooks/config-changed-postupgrade', '')
+        os.execl('./hooks/config-changed-postupgrade',
+                 'config-changed-postupgrade')
 
 if __name__ == '__main__':
     openstack_upgrade()

actions/security-checklist

+security_checklist.py
\ No newline at end of file

actions/security_checklist.py

+#!/usr/bin/env python3
+#
+# Copyright 2019 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import configparser
+import os
+import sys
+
+sys.path.append('.')
+
+import charmhelpers.contrib.openstack.audits as audits
+from charmhelpers.contrib.openstack.audits import (
+    openstack_security_guide,
+)
+
+
+# Via the openstack_security_guide above, we are running the following
+# security assertions automatically:
+#
+# - Check-Identity-01 - validate-file-ownership
+# - Check-Identity-02 - validate-file-permissions
+
+
+@audits.audit(audits.is_audit_type(audits.AuditType.OpenStackSecurityGuide),)
+def uses_sha256_for_hashing_tokens(audit_options):
+    """Validate that SHA256 is used to hash tokens.
+
+    Security Guide Check Name: Check-Identity-04
+
+    :param audit_options: Dictionary of options for audit configuration
+    :type audit_options: Dict
+    :raises: AssertionError if the assertion fails.
+    """
+    section = audit_options['keystone-conf'].get('token')
+    assert section is not None, "Missing section 'token'"
+    provider = section.get('provider')
+    algorithm = section.get("hash_algorithm")
+    if provider and "pki" in provider:
+        assert "SHA256" == algorithm, \
+            "Weak hash algorithm used with PKI provider: ".format(
+                algorithm)
+
+
+@audits.audit(audits.is_audit_type(audits.AuditType.OpenStackSecurityGuide),
+              audits.since_openstack_release('keystone', 'juno'))
+def check_max_request_body_size(audit_options):
+    """Validate that a sane max_request_body_size is set.
+
+    Security Guide Check Name: Check-Identity-05
+
+    :param audit_options: Dictionary of options for audit configuration
+    :type audit_options: Dict
+    :raises: AssertionError if the assertion fails.
+    """
+    default = audit_options['keystone-conf'].get('DEFAULT', {})
+    oslo_middleware = audit_options['keystone-conf'] \
+        .get('oslo_middleware', {})
+    # assert section is not None, "Missing section 'DEFAULT'"
+    assert (default.get('max_request_body_size') or
+            oslo_middleware.get('max_request_body_size') is not None), \
+        "max_request_body_size should be set"
+
+
+@audits.audit(audits.is_audit_type(audits.AuditType.OpenStackSecurityGuide))
+def disable_admin_token(audit_options):
+    """Validate that the admin token is disabled.
+
+    Security Guide Check Name: Check-Identity-06
+
+    :param audit_options: Dictionary of options for audit configuration
+    :type audit_options: Dict
+    :raises: AssertionError if the assertion fails.
+    """
+    default = audit_options['keystone-conf'].get('DEFAULT')
+    assert default is not None, "Missing section 'DEFAULT'"
+    assert default.get('admin_token') is None, \
+        "admin_token should be unset"
+    keystone_paste = _config_file('/etc/keystone/keystone-paste.ini')
+    section = keystone_paste.get('filter:admin_token_auth')
+    if section is not None:
+        assert section.get('AdminTokenAuthMiddleware') is None, \
+            'AdminTokenAuthMiddleware should be unset in keystone-paste.ini'
+
+
+@audits.audit(audits.is_audit_type(audits.AuditType.OpenStackSecurityGuide))
+def insecure_debug_is_false(audit_options):
+    """Valudaite that insecure_debug is false.
+
+    Security Guide Check Name: Check-Identity-07
+
+    :param audit_options: Dictionary of options for audit configuration
+    :type audit_options: Dict
+    :raises: AssertionError if the assertion fails.
+    """
+    section = audit_options['keystone-conf'].get('DEFAULT')
+    assert section is not None, "Missing section 'DEFAULT'"
+    insecure_debug = section.get('insecure_debug')
+    if insecure_debug is not None:
+        assert insecure_debug == "false", \
+            "insecure_debug should be false"
+
+
+@audits.audit(audits.is_audit_type(audits.AuditType.OpenStackSecurityGuide),
+              audits.since_openstack_release('keystone', 'pike'),
+              audits.before_openstack_release('keystone', 'rocky'))
+def uses_fernet_token(audit_options):
+    """Validate that fernet tokens are used.
+
+    Security Guide Check Name: Check-Identity-08
+
+    :param audit_options: Dictionary of options for audit configuration
+    :type audit_options: Dict
+    :raises: AssertionError if the assertion fails.
+    """
+    section = audit_options['keystone-conf'].get('token')
+    assert section is not None, "Missing section 'token'"
+    assert "fernet" == section.get("provider"), \
+        "Fernet tokens are not used"
+
+
+@audits.audit(audits.is_audit_type(audits.AuditType.OpenStackSecurityGuide),
+              audits.since_openstack_release('keystone', 'rocky'))
+def uses_fernet_token_after_default(audit_options):
+    """Validate that fernet tokens are used.
+
+    :param audit_options: Dictionary of options for audit configuration
+    :type audit_options: Dict
+    :raises: AssertionError if the assertion fails.
+    """
+    section = audit_options['keystone-conf'].get('token')
+    assert section is not None, "Missing section 'token'"
+    provider = section.get("provider")
+    if provider:
+        assert "fernet" == provider, "Fernet tokens are not used"
+
+
+def _config_file(path):
+    """Read and parse config file at `path` as an ini file.
+
+    :param path: Path of the file
+    :type path: List[str]
+    :returns: Parsed contents of the file at path
+    :rtype Dict:
+    """
+    conf = configparser.ConfigParser()
+    conf.read(os.path.join(*path))
+    return dict(conf)
+
+
+def main():
+    config = {
+        'config_path': '/etc/keystone',
+        'config_file': 'keystone.conf',
+        'audit_type': audits.AuditType.OpenStackSecurityGuide,
+        'files': openstack_security_guide.FILE_ASSERTIONS['keystone'],
+        'excludes': [
+            'validate-uses-keystone',
+            'validate-uses-tls-for-glance',
+            'validate-uses-tls-for-keystone',
+        ],
+    }
+    config['keystone-conf'] = _config_file(
+        [config['config_path'], config['config_file']])
+    return audits.action_parse_results(audits.run(config))
+
+if __name__ == "__main__":
+    sys.exit(main())

charm-helpers-hooks.yaml

-repo: https://github.com/juju/charm-helpers
+repo: https://github.com/juju/charm-helpers@stable/19.10
 destination: charmhelpers
 include:
     - core
@@ -15,6 +15,7 @@ include:
     - payload
     - contrib.peerstorage
     - contrib.network.ip
-    - contrib.python.packages
+    - contrib.python
     - contrib.charmsupport
     - contrib.hardening|inc=*
+    - contrib.openstack.policyd

charm-helpers-tests.yaml

-repo: https://github.com/juju/charm-helpers
-destination: tests/charmhelpers
-include:
-    - contrib.amulet
-    - contrib.openstack.amulet
-    - core
-    - osplatform

charmhelpers/__init__.py

@@ -23,22 +23,22 @@ import subprocess
 import sys
 
 try:
-    import six  # flake8: noqa
+    import six  # NOQA:F401
 except ImportError:
     if sys.version_info.major == 2:
         subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
     else:
         subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
-    import six  # flake8: noqa
+    import six  # NOQA:F401
 
 try:
-    import yaml  # flake8: noqa
+    import yaml  # NOQA:F401
 except ImportError:
     if sys.version_info.major == 2:
         subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
     else:
         subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
-    import yaml  # flake8: noqa
+    import yaml  # NOQA:F401
 
 
 # Holds a list of mapping of mangled function names that have been deprecated

charmhelpers/cli/unitdata.py

@@ -19,9 +19,16 @@ from charmhelpers.core import unitdata
 @cmdline.subcommand_builder('unitdata', description="Store and retrieve data")
 def unitdata_cmd(subparser):
     nested = subparser.add_subparsers()
+
     get_cmd = nested.add_parser('get', help='Retrieve data')
     get_cmd.add_argument('key', help='Key to retrieve the value of')
     get_cmd.set_defaults(action='get', value=None)
+
+    getrange_cmd = nested.add_parser('getrange', help='Retrieve multiple data')
+    getrange_cmd.add_argument('key', metavar='prefix',
+                              help='Prefix of the keys to retrieve')
+    getrange_cmd.set_defaults(action='getrange', value=None)
+
     set_cmd = nested.add_parser('set', help='Store data')
     set_cmd.add_argument('key', help='Key to set')
     set_cmd.add_argument('value', help='Value to store')
@@ -30,6 +37,8 @@ def unitdata_cmd(subparser):
     def _unitdata_cmd(action, key, value):
         if action == 'get':
             return unitdata.kv().get(key)
+        elif action == 'getrange':
+            return unitdata.kv().getrange(key)
         elif action == 'set':
             unitdata.kv().set(key, value)
             unitdata.kv().flush()