From a9d41b6394ec7a7388bb187913f035cb29ce57b0 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu <ibalutoiu@cloudbasesolutions.com> Date: Wed, 10 Jan 2018 11:32:34 +0000 Subject: [PATCH] Sync plug-in code in the Juju charm --- .../files/plugins/admin-costs/gnocchi.py | 42 +++ .../files/plugins/admin-costs/models.py | 265 ------------------ .../files/plugins/admin-costs/orm.py | 47 ++++ .../files/plugins/admin-costs/utils.py | 68 +++++ .../files/plugins/admin-costs/views.py | 211 +++----------- .../files/plugins/project-costs/gnocchi.py | 42 +++ .../files/plugins/project-costs/models.py | 265 ------------------ .../files/plugins/project-costs/orm.py | 47 ++++ .../files/plugins/project-costs/utils.py | 68 +++++ .../files/plugins/project-costs/views.py | 186 +++--------- charms/garr-dashboard/hooks/hooks.py | 11 +- 11 files changed, 406 insertions(+), 846 deletions(-) create mode 100644 charms/garr-dashboard/files/plugins/admin-costs/gnocchi.py delete mode 100644 charms/garr-dashboard/files/plugins/admin-costs/models.py create mode 100644 charms/garr-dashboard/files/plugins/admin-costs/orm.py create mode 100644 charms/garr-dashboard/files/plugins/admin-costs/utils.py create mode 100644 charms/garr-dashboard/files/plugins/project-costs/gnocchi.py delete mode 100644 charms/garr-dashboard/files/plugins/project-costs/models.py create mode 100644 charms/garr-dashboard/files/plugins/project-costs/orm.py create mode 100644 charms/garr-dashboard/files/plugins/project-costs/utils.py diff --git a/charms/garr-dashboard/files/plugins/admin-costs/gnocchi.py b/charms/garr-dashboard/files/plugins/admin-costs/gnocchi.py new file mode 100644 index 0000000..6303e98 --- /dev/null +++ b/charms/garr-dashboard/files/plugins/admin-costs/gnocchi.py @@ -0,0 +1,42 @@ +from django.conf import settings +from openstack_dashboard.api import base +from openstack_dashboard.api import base +from horizon.utils.memoized import memoized +from gnocchiclient import auth +from gnocchiclient.v1 import client + +class GnocchiTokenNoAuthPlugin(auth.GnocchiNoAuthPlugin): + """No authentication plugin that makes use of the user token + """ + def __init__(self, token_id, endpoint, **kwargs): + user_id = kwargs.get('user_id', 'None') + project_id = kwargs.get('project_id', 'None') + roles = kwargs.get('roles', 'None') + super(GnocchiTokenNoAuthPlugin, self).__init__(user_id, project_id, + roles, endpoint) + self._token = token_id + + def get_headers(self, session, **kwargs): + return {'x-user-id': self._user_id, + 'x-auth-token': self._token, + 'x-project-id': self._project_id + } + + def get_token(self, session, **kwargs): + return self._token + +@memoized +def gnocchi_client(request): + """Initialize Gnocchi client.""" + + endpoint = base.url_for(request, 'metric') + insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', True) + cacert = getattr(settings, 'OPENSTACK_SSL_CACERT', True) + user = request.user + roles = 'default' + auth_plugin = GnocchiTokenNoAuthPlugin(user.token.id, + endpoint, + project_id=user.project_id, + user_id=user.id) + + return client.Client(session_options={'auth': auth_plugin}) diff --git a/charms/garr-dashboard/files/plugins/admin-costs/models.py b/charms/garr-dashboard/files/plugins/admin-costs/models.py deleted file mode 100644 index 4c81f29..0000000 --- a/charms/garr-dashboard/files/plugins/admin-costs/models.py +++ /dev/null @@ -1,265 +0,0 @@ -from __future__ import unicode_literals - -from django.db import models - - -class Charged(models.Model): - id = models.CharField(max_length=100, primary_key=True) - project = models.ForeignKey('Project', blank=True, null=True) - amount = models.FloatField() - refund = models.FloatField(blank=True, null=True) - time = models.DateTimeField() - - class Meta: - managed = False - db_table = 'charged' - - -class Cpu(models.Model): - id = models.CharField(max_length=100, primary_key=True) - brand = models.CharField(max_length=20, blank=True, null=True) - model = models.CharField(max_length=20, blank=True, null=True) - cores = models.IntegerField(unique=True, blank=True, null=True) - speed = models.FloatField(blank=True, null=True) - aggregate = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'cpu' - - -class Event(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=38) - time = models.DateTimeField() - target = models.ForeignKey('Instance', db_column='target', blank=True, null=True) - - class Meta: - managed = False - db_table = 'event' - - -class External(models.Model): - id = models.CharField(max_length=100, primary_key=True) - provider = models.ForeignKey('Provider', blank=True, null=True) - model = models.CharField(max_length=80) - - class Meta: - managed = False - db_table = 'external' - - -class Flavor(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(max_length=20, blank=True, null=True) - cpu = models.IntegerField(blank=True, null=True) - gpu = models.IntegerField(blank=True, null=True) - ram = models.FloatField(blank=True, null=True) - hd = models.IntegerField(blank=True, null=True) - ssd = models.IntegerField(blank=True, null=True) - ephemeral = models.IntegerField() - - class Meta: - managed = False - db_table = 'flavor' - - -class Funds(models.Model): - id = models.CharField(max_length=100, primary_key=True) - source = models.ForeignKey('Organization', blank=True, null=True) - project = models.ForeignKey('Project', blank=True, null=True) - amount = models.FloatField(blank=True, null=True) - time = models.TimeField() - - class Meta: - managed = False - db_table = 'funds' - - -class Gpu(models.Model): - id = models.CharField(max_length=100, primary_key=True) - brand = models.CharField(max_length=40, blank=True, null=True) - model = models.CharField(max_length=20, blank=True, null=True) - aggregate = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'gpu' - - -class HostAggregate(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(max_length=100) - region_id = models.CharField(max_length=40) - - class Meta: - managed = False - db_table = 'host_aggregate' - - -class Instance(models.Model): - id = models.CharField(max_length=100, primary_key=True) - flavor = models.ForeignKey(Flavor, blank=True, null=True) - project = models.ForeignKey('Project', blank=True, null=True) - hd_type = models.IntegerField(blank=True, null=True) - aggregate_id = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'instance' - - -class IpAssociation(models.Model): - id = models.CharField(max_length=100, primary_key=True) - ip = models.CharField(max_length=20) - creation = models.DateTimeField() - termination = models.DateTimeField(blank=True, null=True) - instance = models.ForeignKey(Instance, blank=True, null=True) - - class Meta: - managed = False - db_table = 'ip_association' - - -class Licence(models.Model): - id = models.CharField(max_length=100, primary_key=True) - provider = models.ForeignKey('Provider', blank=True, null=True) - model = models.CharField(max_length=80, blank=True, null=True) - version = models.CharField(max_length=20, blank=True, null=True) - - class Meta: - managed = False - db_table = 'licence' - - -class Meter(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=54) - resource = models.IntegerField(blank=True, null=True) - extra = models.TextField(blank=True, null=True) - project = models.ForeignKey('Project', blank=True, null=True) - quantity = models.FloatField() - time = models.DateTimeField() - duration = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'meter' - - -class Organization(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(max_length=100) - department = models.CharField(max_length=20, blank=True, null=True) - address = models.CharField(max_length=100, blank=True, null=True) - manager = models.ForeignKey('User', db_column='manager') - - class Meta: - managed = False - db_table = 'organization' - - -class Price(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=35, blank=True, null=True) - resource = models.IntegerField(blank=True, null=True) - unit = models.CharField(max_length=32) - price = models.FloatField() - since = models.DateField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'price' - - -class Project(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(unique=True, max_length=255) - os_id = models.CharField(max_length=40) - start = models.DateTimeField() - end = models.DateTimeField(blank=True, null=True) - state = models.IntegerField(blank=True, null=True) - remaining = models.FloatField(blank=True, null=True) - last_update = models.DateTimeField() - - class Meta: - managed = False - db_table = 'project' - - -class Provider(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(max_length=100, blank=True, null=True) - address = models.CharField(max_length=100, blank=True, null=True) - contact = models.CharField(max_length=40, blank=True, null=True) - vat = models.CharField(max_length=20, blank=True, null=True) - - class Meta: - managed = False - db_table = 'provider' - - -class Quotas(models.Model): - id = models.CharField(max_length=100, primary_key=True) - project = models.ForeignKey(Project, blank=True, null=True) - resource = models.CharField(max_length=54) - size = models.IntegerField(blank=True, null=True) - aggregate = models.ForeignKey(HostAggregate, blank=True, null=True) - - class Meta: - managed = False - db_table = 'quotas' - - -class Ram(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=40, blank=True, null=True) - aggregate = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'ram' - - -class Storage(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=80) - volume = models.CharField(max_length=80) - ephemeral = models.IntegerField(blank=True, null=True) - aggregate = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'storage' - - -class User(models.Model): - id = models.CharField(max_length=100, primary_key=True) - os_id = models.CharField(max_length=40) - name = models.CharField(max_length=255) - email = models.CharField(unique=True, max_length=255) - password = models.TextField(blank=True, null=True) - state = models.IntegerField() - cn = models.CharField(max_length=255, blank=True, null=True) - source = models.CharField(max_length=255, blank=True, null=True) - created = models.DateTimeField() - end = models.DateTimeField(blank=True, null=True) - updated = models.DateTimeField(blank=True, null=True) - duration = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'user' - - -class UserProject(models.Model): - id = models.CharField(max_length=100, primary_key=True) - user_id = models.IntegerField() - project_id = models.IntegerField() - admin = models.IntegerField() - start = models.DateTimeField() - end = models.DateTimeField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'user_project' diff --git a/charms/garr-dashboard/files/plugins/admin-costs/orm.py b/charms/garr-dashboard/files/plugins/admin-costs/orm.py new file mode 100644 index 0000000..6edc1af --- /dev/null +++ b/charms/garr-dashboard/files/plugins/admin-costs/orm.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +""" +This file contains the SqlAlchemy mappings to the table in theCloudUsage DB. +""" + +from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, ForeignKey +from sqlalchemy.ext.automap import automap_base +import sqlalchemy.orm + +from django.conf import settings + +try: + from settings import GARR_DATABASE_URL +except: + import os + GARR_DATABASE_URL = os.environ['CLOUDUSAGE_URL'] + +Base = automap_base() + +# reflect +engine = create_engine(GARR_DATABASE_URL) + +# Set up mapped classes and relationships. +Base.prepare(engine, reflect=True) + +# mapped classes are ready +Charged = Base.classes.charged +Cpu = Base.classes.cpu +Event = Base.classes.event +External = Base.classes.external +Flavor = Base.classes.flavor +Funds = Base.classes.funds +Gpu = Base.classes.gpu +HostAggregate = Base.classes.host_aggregate +Instance = Base.classes.instance +IpAssociation = Base.classes.ip_association +Licence = Base.classes.licence +Meter = Base.classes.meter +Organization = Base.classes.organization +Price = Base.classes.price +Project = Base.classes.project +Provider = Base.classes.provider +Quotas = Base.classes.quotas +Ram = Base.classes.ram +Storage = Base.classes.storage +User = Base.classes.user +UserProject = Base.classes.user_project diff --git a/charms/garr-dashboard/files/plugins/admin-costs/utils.py b/charms/garr-dashboard/files/plugins/admin-costs/utils.py new file mode 100644 index 0000000..ef039b6 --- /dev/null +++ b/charms/garr-dashboard/files/plugins/admin-costs/utils.py @@ -0,0 +1,68 @@ +from django.utils.dateparse import parse_datetime +from openstack_dashboard import api +from datetime import datetime + +def convert_nanoseconds(value): + seconds=(value/1000000000)%60 + seconds = int(seconds) + minutes=(value/(1000000000*60))%60 + minutes = int(minutes) + return (value/(1000000000*60*60))%24 + +def get_hours(start_date, end_date): + start = datetime.strptime(start_date, "%Y-%m-%d") + end = datetime.strptime(end_date, "%Y-%m-%d") + diff = end - start + return diff.days * 24 + +def get_measure_value(measures, metric, resource): + if not measures: return 0 + if metric in ['network.outgoing.bytes', 'network.incoming.bytes', 'cpu']: + result = measures[-1] - measures[0] + else: + result = sum(measures) + + if metric == 'cpu': + result = convert_nanoseconds(result) + return result + +def get_volume_usage(request, resource, price, usage_start, usage_end, gnocchi_client): + unit = 'GB' + try: + api.cinder.volume_get(request, resource['id']) + except Exception: + raise Exception('Volume %s not found' % resource['id']) + + measures = [measure[2] for measure in gnocchi_client.metric.get_measures( + 'volume.size', start=usage_start, stop=usage_end, + aggregation='max', resource_id=resource['id'] + )] + + if not measures: + return 0, 0, unit + + usage = sum(measures)/len(measures) + cost = '{0:.2f}'.format( + round(usage * len(measures) * price, 2) + ) + usage = '{0:.2f}'.format(usage) + + return cost, usage, unit + +def get_instance_usage(resource, price, usage_start, usage_end): + unit = 'hour' + start_time = parse_datetime(resource['started_at']).replace(tzinfo=None) + start_query = datetime.strptime(usage_start, "%Y-%m-%d") + diff = start_query - start_time + + hours_interval = get_hours(usage_start, usage_end) + if diff.days < 0: + end_query = datetime.strptime(usage_end, "%Y-%m-%d") + diff = end_query - start_time + hours_interval = diff.days * 24 + + cost = '{0:.2f}'.format( + round(hours_interval * price, 2) + ) + + return cost, hours_interval, unit diff --git a/charms/garr-dashboard/files/plugins/admin-costs/views.py b/charms/garr-dashboard/files/plugins/admin-costs/views.py index cee7f89..605e61c 100644 --- a/charms/garr-dashboard/files/plugins/admin-costs/views.py +++ b/charms/garr-dashboard/files/plugins/admin-costs/views.py @@ -2,72 +2,27 @@ import logging from oslo_utils import units -from django.conf import settings -from django.core.urlresolvers import reverse -from django.core.urlresolvers import reverse_lazy from django.utils.translation import ugettext_lazy as _ -from django.utils.dateparse import parse_datetime from django.utils import timezone - from horizon import exceptions from horizon import messages from horizon import tables -from horizon import views -from horizon import forms -import datetime -from django.utils.dateparse import parse_datetime +from horizon import fogitrms + from openstack_dashboard import api from openstack_dashboard import policy +from sqlalchemy.orm import Session -from openstack_dashboard.api import base -from horizon.utils.memoized import memoized -from gnocchiclient import auth -from gnocchiclient.v1 import client - -from models import Flavor, Price, Storage from openstack_dashboard.dashboards.project.costs import tables as costs_tables +from gnocchi import gnocchi_client +from utils import convert_nanoseconds, get_instance_usage, get_volume_usage +from orm import engine, Flavor, Price, Storage + LOG = logging.getLogger(__name__) GARR_DB_KEY = 'GARR_DATABASE' -class GnocchiTokenNoAuthPlugin(auth.GnocchiNoAuthPlugin): - """No authentication plugin that makes use of the user token - """ - def __init__(self, token_id, endpoint, **kwargs): - user_id = kwargs.get('user_id', 'None') - project_id = kwargs.get('project_id', 'None') - roles = kwargs.get('roles', 'None') - super(GnocchiTokenNoAuthPlugin, self).__init__(user_id, project_id, - roles, endpoint) - self._token = token_id - - def get_headers(self, session, **kwargs): - return {'x-user-id': self._user_id, - 'x-auth-token': self._token, - 'x-project-id': self._project_id - } - - def get_token(self, session, **kwargs): - return self._token - -@memoized -def gnocchi_client(request): - """Initialize Gnocchi client.""" - - endpoint = base.url_for(request, 'metric') - insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', True) - cacert = getattr(settings, 'OPENSTACK_SSL_CACERT', True) - user = request.user - roles = 'default' - auth_plugin = GnocchiTokenNoAuthPlugin(user.token.id, - endpoint, - project_id=user.project_id, - user_id=user.id) - - return client.Client(session_options={'auth': auth_plugin}) - - class IndexView(tables.DataTableView): table_class = costs_tables.CostsTable @@ -79,7 +34,6 @@ class IndexView(tables.DataTableView): context['form'] = self.get_form(self.request) context['overall_cost'] = sum(float(row['cost']) for row in context['table'].data) return context - @staticmethod def get_default_interval(): @@ -97,7 +51,6 @@ class IndexView(tables.DataTableView): end = req.GET.get('end', None) if not start and not end: start, end = self.get_default_interval() - self.form = forms.DateForm(initial={'start': start, 'end': end}) return self.form @@ -112,96 +65,53 @@ class IndexView(tables.DataTableView): def get_data(self): try: - getattr(settings, GARR_DB_KEY) - except AttributeError: - settings.DATABASES[GARR_DB_KEY] = { - 'NAME': settings.GARR_DATABASE_NAME, - 'USER': settings.GARR_DATABASE_USER, - 'HOST': settings.GARR_DATABASE_HOST, - 'PASSWORD': settings.GARR_DATABASE_USER_PASSWORD, - 'ENGINE': settings.GARR_DATABASE_ENGINE - } - - project = self.request.GET.get('project', self.request.user.project_id) + session = Session(engine) + except Exception as e: + LOG.error('Unable to connect to the database') + LOG.error(e) + return [] + project_id = self.request.GET.get('project', self.request.user.project_id) query = { "=": { - "project_id": project + "project_id": project_id } } resource_types = ['instance', 'volume'] - try: - client = gnocchi_client(self.request) - except Exception as ex: - msg = _('Unable to establish gnocchi connection') - messages.error(msg) - exceptions.handle(self.request, msg) - + client = gnocchi_client(self.request) project_resources = [] for resource_type in resource_types: try: resources = client.resource.search(resource_type=resource_type, query=query, details=True) - except Exception as ex: - msg = _('Unable to retrieve gnocchi resources') - exceptions.handle(self.request, msg) + except Exception: + LOG.error('Error while calling gnocchi api') + messages.error(self.request, _("Error while calling Gnocchi API")) + continue project_resources.extend(resources) - + usage_start, usage_end = self.get_interval(self.request) - hours_interval = self.get_hours(usage_start, usage_end) result = [] for resource in project_resources: - resource_type = resource['type'] - resource_id = resource['id'] - name = resource.get('name', None) - display_name = resource.get("display_name", None) - resource_name = name or display_name or resource['id'] - + resource_name = resource.get('name', None) or resource.get('display_name', None) or resource['id'] try: - price, cost_unit = self.get_price(resource) - except Exception as ex: - msg = _('Unable to retrieve resource price') - exceptions.handle(self.request, msg) - - formated_price = '{}/{}'.format(price, cost_unit) - if resource_type == 'instance': - start_time = parse_datetime(resource['started_at']).replace(tzinfo=None) - start_query = datetime.datetime.strptime(usage_start, "%Y-%m-%d") - diff = start_query - start_time - if diff.days < 0: - end_query = datetime.datetime.strptime(usage_end, "%Y-%m-%d") - diff = end_query - start_time - hours_interval = diff.days * 24 - - usage = hours_interval - cost = '{0:.2f}'.format( - round(usage * price, 2) - ) - unit = 'hour' - elif resource_type == 'volume': - try: - api.cinder.volume_get(self.request, resource['id']) - except Exception: - LOG.error('Unable to find volume with the following id %s' % resource['id']) - continue - try: - measures = [measure[2] for measure in client.metric.get_measures( - 'volume.size', start=usage_start, stop=usage_end, - aggregation='max', resource_id=resource_id - )] - except Exception as ex: - msg = _('Unable to retrieve measurements for volumes') - exceptions.handle(self.request, msg) + price, cost_unit = self.get_price(resource, self.request, session) + except Exception as e: + LOG.error("Unable to fetch price for resource %s of type %s" % (resource_name, resource['type'])) + continue + + if type(cost_unit) is set: + cost_unit = cost_unit.pop() - if not measures: + try: + if resource['type'] == 'instance': + cost, usage, unit = get_instance_usage(resource, price, usage_start, usage_end) + elif resource['type'] == 'volume': + cost, usage, unit = get_volume_usage(self.request, resource, price, usage_start, usage_end, client) + else: continue - usage = sum(measures)/len(measures) - cost = '{0:.2f}'.format( - round(usage * len(measures) * price, 2) - ) - usage = '{0:.2f}'.format(usage) - unit = 'GB' - else: + except Exception as ex: + LOG.error('Unable to get usage for resource %s' % resource) continue result.append( { @@ -209,54 +119,23 @@ class IndexView(tables.DataTableView): 'unit': unit, 'value': usage, 'id': resource['id'], - 'price': formated_price, - 'resource_type': resource_type, + 'price': '{}/{}'.format(price, cost_unit), + 'resource_type': resource['type'], 'cost': cost }) - return sorted(result, key=lambda resource: resource['name']) - - def get_measure_value(self, measures, metric, resource): - if not measures: return 0 - if metric in ['network.outgoing.bytes', 'neZtwork.incoming.bytes', 'cpu']: - result = measures[-1] - measures[0] - else: - result = sum(measures) - if metric == 'cpu': - result = self.convert_nanoseconds(result) - - return result + return sorted(result, key=lambda resource: resource['name']) - @staticmethod - def get_hours(start_date, end_date): - start = datetime.datetime.strptime(start_date, "%Y-%m-%d") - end = datetime.datetime.strptime(end_date, "%Y-%m-%d") - diff = end - start - return diff.days * 24 - def get_price(self, resource): - price = None + def get_price(self, resource, request, db_session): if resource['type'] == 'instance': flavor_name = resource.get('flavor_name', None) if not flavor_name: flavor_name = api.nova.flavor_get(request, resource['flavor_id']).name - flavor = Flavor.objects.using(GARR_DB_KEY).get(name=flavor_name) - - price = Price.objects.using(GARR_DB_KEY).get(resource=flavor.id, type='flavor') + flavor = db_session.query(Flavor).filter(Flavor.name == flavor_name).first() + price = db_session.query(Price).filter(Price.resource == flavor.id, Price.type == 'flavor').first() elif resource['type'] == 'volume': volume_type = resource['volume_type'] or '' - try: - volume = Storage.objects.using(GARR_DB_KEY).get(volume=volume_type) - price = Price.objects.using(GARR_DB_KEY).get(resource=volume.id, type='storage') - except Exception as e: - LOG.error('Unable to retrieve volume of type %s' % volume_type) - LOG.error(e) - raise Exception(e) + volume = db_session.query(Storage).filter(Storage.volume == volume_type).first() + price = db_session.query(Price).filter(Price.resource == volume.id, Price.type == 'storage').first() + return price.price, price.unit - - @staticmethod - def convert_nanoseconds(value): - seconds=(value/1000000000)%60 - seconds = int(seconds) - minutes=(value/(1000000000*60))%60 - minutes = int(minutes) - return (value/(1000000000*60*60))%24 diff --git a/charms/garr-dashboard/files/plugins/project-costs/gnocchi.py b/charms/garr-dashboard/files/plugins/project-costs/gnocchi.py new file mode 100644 index 0000000..6303e98 --- /dev/null +++ b/charms/garr-dashboard/files/plugins/project-costs/gnocchi.py @@ -0,0 +1,42 @@ +from django.conf import settings +from openstack_dashboard.api import base +from openstack_dashboard.api import base +from horizon.utils.memoized import memoized +from gnocchiclient import auth +from gnocchiclient.v1 import client + +class GnocchiTokenNoAuthPlugin(auth.GnocchiNoAuthPlugin): + """No authentication plugin that makes use of the user token + """ + def __init__(self, token_id, endpoint, **kwargs): + user_id = kwargs.get('user_id', 'None') + project_id = kwargs.get('project_id', 'None') + roles = kwargs.get('roles', 'None') + super(GnocchiTokenNoAuthPlugin, self).__init__(user_id, project_id, + roles, endpoint) + self._token = token_id + + def get_headers(self, session, **kwargs): + return {'x-user-id': self._user_id, + 'x-auth-token': self._token, + 'x-project-id': self._project_id + } + + def get_token(self, session, **kwargs): + return self._token + +@memoized +def gnocchi_client(request): + """Initialize Gnocchi client.""" + + endpoint = base.url_for(request, 'metric') + insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', True) + cacert = getattr(settings, 'OPENSTACK_SSL_CACERT', True) + user = request.user + roles = 'default' + auth_plugin = GnocchiTokenNoAuthPlugin(user.token.id, + endpoint, + project_id=user.project_id, + user_id=user.id) + + return client.Client(session_options={'auth': auth_plugin}) diff --git a/charms/garr-dashboard/files/plugins/project-costs/models.py b/charms/garr-dashboard/files/plugins/project-costs/models.py deleted file mode 100644 index 4c81f29..0000000 --- a/charms/garr-dashboard/files/plugins/project-costs/models.py +++ /dev/null @@ -1,265 +0,0 @@ -from __future__ import unicode_literals - -from django.db import models - - -class Charged(models.Model): - id = models.CharField(max_length=100, primary_key=True) - project = models.ForeignKey('Project', blank=True, null=True) - amount = models.FloatField() - refund = models.FloatField(blank=True, null=True) - time = models.DateTimeField() - - class Meta: - managed = False - db_table = 'charged' - - -class Cpu(models.Model): - id = models.CharField(max_length=100, primary_key=True) - brand = models.CharField(max_length=20, blank=True, null=True) - model = models.CharField(max_length=20, blank=True, null=True) - cores = models.IntegerField(unique=True, blank=True, null=True) - speed = models.FloatField(blank=True, null=True) - aggregate = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'cpu' - - -class Event(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=38) - time = models.DateTimeField() - target = models.ForeignKey('Instance', db_column='target', blank=True, null=True) - - class Meta: - managed = False - db_table = 'event' - - -class External(models.Model): - id = models.CharField(max_length=100, primary_key=True) - provider = models.ForeignKey('Provider', blank=True, null=True) - model = models.CharField(max_length=80) - - class Meta: - managed = False - db_table = 'external' - - -class Flavor(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(max_length=20, blank=True, null=True) - cpu = models.IntegerField(blank=True, null=True) - gpu = models.IntegerField(blank=True, null=True) - ram = models.FloatField(blank=True, null=True) - hd = models.IntegerField(blank=True, null=True) - ssd = models.IntegerField(blank=True, null=True) - ephemeral = models.IntegerField() - - class Meta: - managed = False - db_table = 'flavor' - - -class Funds(models.Model): - id = models.CharField(max_length=100, primary_key=True) - source = models.ForeignKey('Organization', blank=True, null=True) - project = models.ForeignKey('Project', blank=True, null=True) - amount = models.FloatField(blank=True, null=True) - time = models.TimeField() - - class Meta: - managed = False - db_table = 'funds' - - -class Gpu(models.Model): - id = models.CharField(max_length=100, primary_key=True) - brand = models.CharField(max_length=40, blank=True, null=True) - model = models.CharField(max_length=20, blank=True, null=True) - aggregate = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'gpu' - - -class HostAggregate(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(max_length=100) - region_id = models.CharField(max_length=40) - - class Meta: - managed = False - db_table = 'host_aggregate' - - -class Instance(models.Model): - id = models.CharField(max_length=100, primary_key=True) - flavor = models.ForeignKey(Flavor, blank=True, null=True) - project = models.ForeignKey('Project', blank=True, null=True) - hd_type = models.IntegerField(blank=True, null=True) - aggregate_id = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'instance' - - -class IpAssociation(models.Model): - id = models.CharField(max_length=100, primary_key=True) - ip = models.CharField(max_length=20) - creation = models.DateTimeField() - termination = models.DateTimeField(blank=True, null=True) - instance = models.ForeignKey(Instance, blank=True, null=True) - - class Meta: - managed = False - db_table = 'ip_association' - - -class Licence(models.Model): - id = models.CharField(max_length=100, primary_key=True) - provider = models.ForeignKey('Provider', blank=True, null=True) - model = models.CharField(max_length=80, blank=True, null=True) - version = models.CharField(max_length=20, blank=True, null=True) - - class Meta: - managed = False - db_table = 'licence' - - -class Meter(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=54) - resource = models.IntegerField(blank=True, null=True) - extra = models.TextField(blank=True, null=True) - project = models.ForeignKey('Project', blank=True, null=True) - quantity = models.FloatField() - time = models.DateTimeField() - duration = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'meter' - - -class Organization(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(max_length=100) - department = models.CharField(max_length=20, blank=True, null=True) - address = models.CharField(max_length=100, blank=True, null=True) - manager = models.ForeignKey('User', db_column='manager') - - class Meta: - managed = False - db_table = 'organization' - - -class Price(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=35, blank=True, null=True) - resource = models.IntegerField(blank=True, null=True) - unit = models.CharField(max_length=32) - price = models.FloatField() - since = models.DateField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'price' - - -class Project(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(unique=True, max_length=255) - os_id = models.CharField(max_length=40) - start = models.DateTimeField() - end = models.DateTimeField(blank=True, null=True) - state = models.IntegerField(blank=True, null=True) - remaining = models.FloatField(blank=True, null=True) - last_update = models.DateTimeField() - - class Meta: - managed = False - db_table = 'project' - - -class Provider(models.Model): - id = models.CharField(max_length=100, primary_key=True) - name = models.CharField(max_length=100, blank=True, null=True) - address = models.CharField(max_length=100, blank=True, null=True) - contact = models.CharField(max_length=40, blank=True, null=True) - vat = models.CharField(max_length=20, blank=True, null=True) - - class Meta: - managed = False - db_table = 'provider' - - -class Quotas(models.Model): - id = models.CharField(max_length=100, primary_key=True) - project = models.ForeignKey(Project, blank=True, null=True) - resource = models.CharField(max_length=54) - size = models.IntegerField(blank=True, null=True) - aggregate = models.ForeignKey(HostAggregate, blank=True, null=True) - - class Meta: - managed = False - db_table = 'quotas' - - -class Ram(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=40, blank=True, null=True) - aggregate = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'ram' - - -class Storage(models.Model): - id = models.CharField(max_length=100, primary_key=True) - type = models.CharField(max_length=80) - volume = models.CharField(max_length=80) - ephemeral = models.IntegerField(blank=True, null=True) - aggregate = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'storage' - - -class User(models.Model): - id = models.CharField(max_length=100, primary_key=True) - os_id = models.CharField(max_length=40) - name = models.CharField(max_length=255) - email = models.CharField(unique=True, max_length=255) - password = models.TextField(blank=True, null=True) - state = models.IntegerField() - cn = models.CharField(max_length=255, blank=True, null=True) - source = models.CharField(max_length=255, blank=True, null=True) - created = models.DateTimeField() - end = models.DateTimeField(blank=True, null=True) - updated = models.DateTimeField(blank=True, null=True) - duration = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'user' - - -class UserProject(models.Model): - id = models.CharField(max_length=100, primary_key=True) - user_id = models.IntegerField() - project_id = models.IntegerField() - admin = models.IntegerField() - start = models.DateTimeField() - end = models.DateTimeField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'user_project' diff --git a/charms/garr-dashboard/files/plugins/project-costs/orm.py b/charms/garr-dashboard/files/plugins/project-costs/orm.py new file mode 100644 index 0000000..e31ef78 --- /dev/null +++ b/charms/garr-dashboard/files/plugins/project-costs/orm.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +""" +This file contains the SqlAlchemy mappings to the table in theCloudUsage DB. +""" + +from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, ForeignKey +from sqlalchemy.ext.automap import automap_base +import sqlalchemy.orm + +from django.conf import settings + +try: + from settings import GARR_DATABASE_URL +except: + import os + GARR_DATABASE_URL = os.environ['CLOUDUSAGE_URL'] + +Base = automap_base() + +# reflect +engine = create_engine(GARR_DATABASE_URL) + +# Set up mapped classes and relationships. +Base.prepare(engine, reflect=True) + +# mapped classes are ready +Charged = Base.classes.charged +Cpu = Base.classes.cpu +Event = Base.classes.event +External = Base.classes.external +Flavor = Base.classes.flavor +Funds = Base.classes.funds +Gpu = Base.classes.gpu +HostAggregate = Base.classes.host_aggregate +Instance = Base.classes.instance +IpAssociation = Base.classes.ip_association +Licence = Base.classes.licence +Meter = Base.classes.meter +Organization = Base.classes.organization +Price = Base.classes.price +Project = Base.classes.project +Provider = Base.classes.provider +Quotas = Base.classes.quotas +Ram = Base.classes.ram +Storage = Base.classes.storage +User = Base.classes.user +UserProject = Base.classes.user_project diff --git a/charms/garr-dashboard/files/plugins/project-costs/utils.py b/charms/garr-dashboard/files/plugins/project-costs/utils.py new file mode 100644 index 0000000..ef039b6 --- /dev/null +++ b/charms/garr-dashboard/files/plugins/project-costs/utils.py @@ -0,0 +1,68 @@ +from django.utils.dateparse import parse_datetime +from openstack_dashboard import api +from datetime import datetime + +def convert_nanoseconds(value): + seconds=(value/1000000000)%60 + seconds = int(seconds) + minutes=(value/(1000000000*60))%60 + minutes = int(minutes) + return (value/(1000000000*60*60))%24 + +def get_hours(start_date, end_date): + start = datetime.strptime(start_date, "%Y-%m-%d") + end = datetime.strptime(end_date, "%Y-%m-%d") + diff = end - start + return diff.days * 24 + +def get_measure_value(measures, metric, resource): + if not measures: return 0 + if metric in ['network.outgoing.bytes', 'network.incoming.bytes', 'cpu']: + result = measures[-1] - measures[0] + else: + result = sum(measures) + + if metric == 'cpu': + result = convert_nanoseconds(result) + return result + +def get_volume_usage(request, resource, price, usage_start, usage_end, gnocchi_client): + unit = 'GB' + try: + api.cinder.volume_get(request, resource['id']) + except Exception: + raise Exception('Volume %s not found' % resource['id']) + + measures = [measure[2] for measure in gnocchi_client.metric.get_measures( + 'volume.size', start=usage_start, stop=usage_end, + aggregation='max', resource_id=resource['id'] + )] + + if not measures: + return 0, 0, unit + + usage = sum(measures)/len(measures) + cost = '{0:.2f}'.format( + round(usage * len(measures) * price, 2) + ) + usage = '{0:.2f}'.format(usage) + + return cost, usage, unit + +def get_instance_usage(resource, price, usage_start, usage_end): + unit = 'hour' + start_time = parse_datetime(resource['started_at']).replace(tzinfo=None) + start_query = datetime.strptime(usage_start, "%Y-%m-%d") + diff = start_query - start_time + + hours_interval = get_hours(usage_start, usage_end) + if diff.days < 0: + end_query = datetime.strptime(usage_end, "%Y-%m-%d") + diff = end_query - start_time + hours_interval = diff.days * 24 + + cost = '{0:.2f}'.format( + round(hours_interval * price, 2) + ) + + return cost, hours_interval, unit diff --git a/charms/garr-dashboard/files/plugins/project-costs/views.py b/charms/garr-dashboard/files/plugins/project-costs/views.py index d7deb59..6c1c5fc 100644 --- a/charms/garr-dashboard/files/plugins/project-costs/views.py +++ b/charms/garr-dashboard/files/plugins/project-costs/views.py @@ -2,73 +2,28 @@ import logging from oslo_utils import units -from django.conf import settings -from django.core.urlresolvers import reverse -from django.core.urlresolvers import reverse_lazy from django.utils.translation import ugettext_lazy as _ -from django.utils.dateparse import parse_datetime from django.utils import timezone from horizon import exceptions from horizon import messages from horizon import tables -from horizon import views from horizon import forms -import datetime -from django.utils.dateparse import parse_datetime + from openstack_dashboard import api from openstack_dashboard import policy +from sqlalchemy.orm import Session -from openstack_dashboard.api import base -from horizon.utils.memoized import memoized -from gnocchiclient import auth -from gnocchiclient.v1 import client - -from models import Flavor, Price, Storage from openstack_dashboard.dashboards.project.costs import tables as costs_tables +from gnocchi import gnocchi_client +from utils import convert_nanoseconds, get_instance_usage, get_volume_usage +from orm import engine, Flavor, Price, Storage + LOG = logging.getLogger(__name__) GARR_DB_KEY = 'GARR_DATABASE' -class GnocchiTokenNoAuthPlugin(auth.GnocchiNoAuthPlugin): - """No authentication plugin that makes use of the user token - """ - def __init__(self, token_id, endpoint, **kwargs): - user_id = kwargs.get('user_id', 'None') - project_id = kwargs.get('project_id', 'None') - roles = kwargs.get('roles', 'None') - super(GnocchiTokenNoAuthPlugin, self).__init__(user_id, project_id, - roles, endpoint) - self._token = token_id - - def get_headers(self, session, **kwargs): - return {'x-user-id': self._user_id, - 'x-auth-token': self._token, - 'x-project-id': self._project_id - } - - def get_token(self, session, **kwargs): - return self._token - -@memoized -def gnocchi_client(request): - """Initialize Gnocchi client.""" - - endpoint = base.url_for(request, 'metric') - insecure = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', True) - cacert = getattr(settings, 'OPENSTACK_SSL_CACERT', True) - user = request.user - roles = 'default' - auth_plugin = GnocchiTokenNoAuthPlugin(user.token.id, - endpoint, - project_id=user.project_id, - user_id=user.id) - - return client.Client(session_options={'auth': auth_plugin}) - - - class IndexView(tables.DataTableView): table_class = costs_tables.CostsTable page_title = _("Project Costs") @@ -80,7 +35,6 @@ class IndexView(tables.DataTableView): context['overall_cost'] = sum(float(row['cost']) for row in context['table'].data) return context - @staticmethod def get_default_interval(): start_time = timezone.now() - timezone.timedelta(days=1) @@ -111,15 +65,11 @@ class IndexView(tables.DataTableView): def get_data(self): try: - getattr(settings, GARR_DB_KEY) - except AttributeError: - settings.DATABASES[GARR_DB_KEY] = { - 'NAME': settings.GARR_DATABASE_NAME, - 'USER': settings.GARR_DATABASE_USER, - 'HOST': settings.GARR_DATABASE_HOST, - 'PASSWORD': settings.GARR_DATABASE_USER_PASSWORD, - 'ENGINE': settings.GARR_DATABASE_ENGINE - } + session = Session(engine) + except Exception as e: + LOG.error('Unable to connect to the database') + LOG.error(e) + return [] query = { "=": { @@ -131,111 +81,61 @@ class IndexView(tables.DataTableView): client = gnocchi_client(self.request) project_resources = [] for resource_type in resource_types: - resources = client.resource.search(resource_type=resource_type, query=query, details=True) + try: + resources = client.resource.search(resource_type=resource_type, query=query, details=True) + except Exception: + LOG.error('Error while calling gnocchi api') + messages.error(self.request, _("Error while calling Gnocchi API")) + continue + project_resources.extend(resources) usage_start, usage_end = self.get_interval(self.request) - hours_interval = self.get_hours(usage_start, usage_end) result = [] for resource in project_resources: - resource_type = resource['type'] - resource_id = resource['id'] - name = resource.get('name', None) - display_name = resource.get("display_name", None) - resource_name = name or display_name or resource['id'] + resource_name = resource.get('name', None) or resource.get('display_name', None) or resource['id'] try: - price, cost_unit = self.get_price(resource, self.request) + price, cost_unit = self.get_price(resource, self.request, session) except Exception as e: - LOG.error("Unable to fetch price for resource %s of type %s" % (resource_name, resource_type)) - LOG.error(e) + LOG.error("Unable to fetch price for resource %s of type %s" % (resource_name, resource['type'])) continue - formated_price = '{}/{}'.format(price, cost_unit) - if resource_type == 'instance': - start_time = parse_datetime(resource['started_at']).replace(tzinfo=None) - start_query = datetime.datetime.strptime(usage_start, "%Y-%m-%d") - diff = start_query - start_time - if diff.days < 0: - end_query = datetime.datetime.strptime(usage_end, "%Y-%m-%d") - diff = end_query - start_time - hours_interval = diff.days * 24 - - usage = hours_interval - cost = '{0:.2f}'.format( - round(usage * price, 2) - ) - unit = 'hour' - elif resource_type == 'volume': - try: - api.cinder.volume_get(self.request, resource['id']) - except Exception: - LOG.error('Unable to find volume with the following id %s' % resource['id']) - continue - measures = [measure[2] for measure in client.metric.get_measures( - 'volume.size', start=usage_start, stop=usage_end, - aggregation='max', resource_id=resource_id - )] - if not measures: + + if type(cost_unit) is set: + cost_unit = cost_unit.pop() + + try: + if resource['type'] == 'instance': + cost, usage, unit = get_instance_usage(resource, price, usage_start, usage_end) + elif resource['type'] == 'volume': + cost, usage, unit = get_volume_usage(self.request, resource, price, usage_start, usage_end, client) + else: continue - usage = sum(measures)/len(measures) - cost = '{0:.2f}'.format( - round(usage * len(measures) * price, 2) - ) - usage = '{0:.2f}'.format(usage) - unit = 'GB' - else: + except Exception as ex: + LOG.error('Unable to get usage for resource %s' % resource) continue + result.append( { 'name': resource_name, 'unit': unit, 'value': usage, 'id': resource['id'], - 'price': formated_price, - 'resource_type': resource_type, + 'price': '{}/{}'.format(price, cost_unit), + 'resource_type': resource['type'], 'cost': cost }) - return sorted(result, key=lambda resource: resource['name']) - def get_measure_value(self, measures, metric, resource): - if not measures: return 0 - if metric in ['network.outgoing.bytes', 'neZtwork.incoming.bytes', 'cpu']: - result = measures[-1] - measures[0] - else: - result = sum(measures) + return sorted(result, key=lambda resource: resource['name']) - if metric == 'cpu': - result = self.convert_nanoseconds(result) - return result - @staticmethod - def get_hours(start_date, end_date): - start = datetime.datetime.strptime(start_date, "%Y-%m-%d") - end = datetime.datetime.strptime(end_date, "%Y-%m-%d") - diff = end - start - return diff.days * 24 - - def get_price(self, resource, request): - price = None + def get_price(self, resource, request, db_session): if resource['type'] == 'instance': flavor_name = resource.get('flavor_name', None) if not flavor_name: flavor_name = api.nova.flavor_get(request, resource['flavor_id']).name - flavor = Flavor.objects.using(GARR_DB_KEY).get(name=flavor_name) - - price = Price.objects.using(GARR_DB_KEY).get(resource=flavor.id, type='flavor') + flavor = db_session.query(Flavor).filter(Flavor.name == flavor_name).first() + price = db_session.query(Price).filter(Price.resource == flavor.id, Price.type == 'flavor').first() elif resource['type'] == 'volume': volume_type = resource['volume_type'] or '' - try: - volume = Storage.objects.using(GARR_DB_KEY).get(volume=volume_type) - price = Price.objects.using(GARR_DB_KEY).get(resource=volume.id, type='storage') - except Exception as e: - LOG.error('Unable to retrieve volume of type %s' % volume_type) - LOG.error(e) - raise Exception(e) + volume = db_session.query(Storage).filter(Storage.volume == volume_type).first() + price = db_session.query(Price).filter(Price.resource == volume.id, Price.type == 'storage').first() + return price.price, price.unit - - @staticmethod - def convert_nanoseconds(value): - seconds=(value/1000000000)%60 - seconds = int(seconds) - minutes=(value/(1000000000*60))%60 - minutes = int(minutes) - return (value/(1000000000*60*60))%24 diff --git a/charms/garr-dashboard/hooks/hooks.py b/charms/garr-dashboard/hooks/hooks.py index 6676c09..92cbe13 100644 --- a/charms/garr-dashboard/hooks/hooks.py +++ b/charms/garr-dashboard/hooks/hooks.py @@ -123,15 +123,11 @@ def config_changed(): def dashboard_plugin_relation_joined(): cfg = config() local_settings = ( - "GARR_DATABASE_NAME=\"%s\"\n" - "GARR_DATABASE_USER=\"%s\"\n" - "GARR_DATABASE_USER_PASSWORD=\"%s\"\n" - "GARR_DATABASE_HOST=\"%s\"\n" - "GARR_DATABASE_ENGINE=\"django.db.backends.mysql\"\n" % ( - cfg["database-name"], + "GARR_DATABASE_URL=\"mysql://%s:%s@%s/%s\"\n" % ( cfg["database-user"], cfg["database-user-password"], - cfg["database-host"] + cfg["database-host"], + cfg["database-name"] ) ) relation_settings = { @@ -152,5 +148,6 @@ def dashboard_plugin_relation_changed(): install_admin_costs_plugin(ctxt['openstack_dir']) install_theme(ctxt['openstack_dir']) pip_install('gnocchiclient==3.3.1', fatal=True) + pip_install('sqlalchemy==1.2.0', fatal=True) service_restart('apache2') status_set('active', 'Unit is ready') -- GitLab