diff --git a/bundle-noceph.yaml b/bundle-noceph.yaml
index 7c45d43156d8a7cb94396787257b578b238cb905..0811a44843f378f3f31b0f88abebf96595ec28d0 100644
--- a/bundle-noceph.yaml
+++ b/bundle-noceph.yaml
@@ -5,7 +5,7 @@
##
series: bionic
-description: A four-machine Kubernetes cluster, appropriate for production. Includes a three-machine etcd and ceph-mon cluster, a Kubernetes master, three Kubernetes worker nodes and a Kubernetes worker with GPUs. The Ceph (Luminous) cluster uses three nodes for Ceph Monitors and a Ceph OSD in LXD containers on each worker machine. Two Ceph MDSs provide redundant shared Ceph File System.
+description: A 9-machine Kubernetes cluster, appropriate for production. Includes a three-machine etcd and ceph-mon cluster, two Kubernetes masters, three Kubernetes worker nodes and a Kubernetes worker with GPUs. The Ceph (Luminous) cluster uses three nodes for Ceph Monitors and a Ceph OSD in LXD containers on each worker machine. Two Ceph MDSs provide redundant shared Ceph File System.
# ----------------------------------------------------------------------
machines:
@@ -30,17 +30,24 @@ machines:
# ----------------------------------------------------------------------
services:
-# defaultgw:
-# charm: cs:~csd-garr/defaultgw
-# options:
-# # ip route | grep default
-# gateway: 90.147.159.1/25
+# canal:
+# annotations:
+# gui-x: '450'
+# gui-y: '750'
+# charm: cs:~containers/canal
+# resources:
+# calico: 73
+# calico-arm64: 69
+# flannel: 73
+# flannel-arm64: 72
easyrsa:
charm: cs:~containers/easyrsa
bindings:
"": space-os-mgmt
num_units: 1
+ resources:
+ easyrsa: 5
to:
# an lxd on the unit of kubernetes-master
- "lxd:kubernetes-master"
@@ -56,6 +63,9 @@ services:
# constraints: tags=kubernetes
options:
channel: 3.2/stable
+ resources:
+ etcd: 3
+ snapshot: 0
to:
- "lxd:0"
- "lxd:1"
@@ -81,6 +91,7 @@ services:
apiserver: space-k8s-pub
expose: true
num_units: 1
+ resources: {}
to:
- "lxd:kubernetes-master"
annotations:
@@ -95,19 +106,26 @@ services:
kubernetes-master:
charm: cs:~containers/kubernetes-master
+ constraints: tags=kubernetes-master
bindings:
"": space-os-mgmt
#kube-api-endpoint: space-pub
num_units: 1
- constraints: tags=kubernetes-master cores=2 mem=4G #root-disk=16G
options:
- channel: 1.11/stable
- authorization-mode: "Node,RBAC,Webhook"
+ channel: 1.13/stable
+ authorization-mode: "Node,RBAC,WebHook"
# RBAC to allow access to dashboard to User "system:anonymous"
# Because of this bug https://github.com/kubernetes/kubernetes/issues/39722
# the value must be false, rather than what the documentation says.
api-extra-args: "anonymous-auth=false"
enable-dashboard-addons: True
+ resources:
+ cdk-addons: 0
+ kube-apiserver: 0
+ kube-controller-manager: 0
+ kube-proxy: 0
+ kube-scheduler: 0
+ kubectl: 0
annotations:
gui-x: '800'
gui-y: '850'
@@ -116,13 +134,13 @@ services:
kubernetes-worker:
charm: cs:~containers/kubernetes-worker
+ constraints: tags=kubernetes cores=4 mem=4G root-disk=16G
bindings:
"": space-os-mgmt
- num_units: 3
- constraints: tags=kubernetes-worker cores=4 mem=4G #root-disk=16G
expose: true
+ num_units: 3
options:
- channel: 1.11/stable
+ channel: 1.13/stable
# Enable Cloud Controller Manager
# https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/
# kubelet-extra-args: --cloud-provider=external
@@ -134,40 +152,16 @@ services:
- "2"
- "3"
-
-
-
-# ceph-proxy:
-# charm: "cs:ceph-proxy"
-# num_units: 1
-# bindings:
-# '': *network_space_default
-# options:
-## admin-key: AQAYOl5Zy5oPABAAGm3E5rQ11CWEs7etRU1Tqg==
-## fsid: "a5256fb9-3bba-4c19-bc1b-2e34fc132706"
-## monitor-hosts: "10.2.3.23:6789,10.2.3.22:6789,10.2.3.21:6789"
-# admin-key: AQAHQ4RVLRCZGxAAawt6yF14NEnSiZKk2MfyBA==
-# fsid: "9a9eedd0-9400-488e-96de-c349fffad7c4"
-# monitor-hosts: "10.2.3.243:6789,10.2.3.242:6789,10.2.3.241:6789"
-# annotations:
-# gui-x: "1300"
-# gui-y: "0"
-# to:
-# - "lxd:0"
-
-
-
kubernetes-worker-gpu:
charm: cs:~containers/kubernetes-worker
bindings:
"": space-os-mgmt
- # charm: ./kubernetes-worker # patched for cuda-9.0
num_units: 1
- constraints: tags=gpu cores=4 mem=4G #root-disk=16G
+ constraints: tags=gpu
expose: true
options:
- channel: 1.11/stable
- cuda_repo: 10.0.130-1
+ channel: 1.13/stable
+ # cuda_repo: 10.0.130-1
# kubelet-extra-args: --cloud-provider=external
annotations:
gui-x: "520"
@@ -175,83 +169,16 @@ services:
to:
- "4"
-# ceph-mon:
-# charm: cs:ceph-mon
-# num_units: 3
-# bindings:
-# "": space-os-mgmt
-# options:
-# # source: cloud:xenial-pike
-# to:
-# - 'lxd:0' # lxd:etcd
-# - 'lxd:1'
-# - 'lxd:2'
-# annotations:
-# gui-x: '1200'
-# gui-y: '400'
-
-# ceph-osd:
-# charm: cs:~csd-garr/ceph-osd
-# num_units: 3
-# bindings:
-# "": space-os-mgmt
-# cluster: space-os-data
-# options:
-# osd-devices: /dev/mapper/mpatha /dev/mapper/mpathb /dev/mapper/mpathc /dev/mapper/mpathd
-# osd-max-backfills: 1
-# osd-recovery-max-active: 1
-# osd-journal-size: 10000
-# use-syslog: True
-# # source: cloud:xenial-pike
-# # bluestore: True
-# to:
-# - "kubernetes-worker/0"
-# - "kubernetes-worker/1"
-# - "kubernetes-worker/2"
-# annotations:
-# gui-x: '1400'
-# gui-y: '400'
-
-# ceph-mds:
-# charm: cs:ceph-fs
-# num_units: 1
-# bindings:
-# "": space-os-mgmt
-# to:
-# - 'lxd:1'
-# annotations:
-# gui-x: '1200'
-# gui-y: '400'
-
- # manila:
- # charm: cs:manila
- # bindings:
- # "": space-os-mgmt
- # to:
- # - 'lxd:2'
- # options:
- # debug: True
-
- # manila-generic:
- # charm: cs:manila-generic
- # bindings:
- # "": space-os-mgmt
- # to:
- # - 'lxd:2'
- # options:
- # debug: True
# ----------------------------------------------------------------------
relations:
- ["kubernetes-master:kube-api-endpoint", "kubeapi-load-balancer:apiserver"]
- ["kubernetes-master:loadbalancer", "kubeapi-load-balancer:loadbalancer"]
- ["kubernetes-master:certificates", "easyrsa:client"]
-# - ["kubernetes-master:juju-info", "defaultgw:juju-info"]
- ["kubernetes-master:juju-info", "kubernetes-keystone:juju-info"]
- ["kubernetes-master:etcd", "etcd:db"]
# Kube load balancer
- ["kubeapi-load-balancer:certificates", "easyrsa:client"]
-# - ["kubeapi-load-balancer:juju-info", "defaultgw:juju-info"]
- ["etcd:certificates", "easyrsa:client"]
# Kube worker
- ["kubernetes-master:kube-control", "kubernetes-worker:kube-control"]
@@ -261,19 +188,14 @@ relations:
- ["kubernetes-master:kube-control", "kubernetes-worker-gpu:kube-control"]
- ["kubernetes-worker-gpu:certificates", "easyrsa:client"]
- ["kubernetes-worker-gpu:kube-api-endpoint", "kubeapi-load-balancer:website"]
+ # Canal
+# - ["canal:etcd", "etcd:db"]
+# - ["canal:cni", "kubernetes-master:cni"]
+# - ["canal:cni", "kubernetes-worker:cni"]
+# - ["canal:cni", "kubernetes-worker-gpu:cni"]
# Flannel
- ["flannel:etcd", "etcd:db"]
- ["flannel:cni", "kubernetes-master:cni"]
- ["flannel:cni", "kubernetes-worker:cni"]
- ["flannel:cni", "kubernetes-worker-gpu:cni"]
- # Ceph
-# - ["kubernetes-master:ceph-storage", "ceph-mon:admin"]
-# - ["kubernetes-master:ceph-storage", "ceph-proxy:client"]
-
-# - ["ceph-osd:mon", "ceph-mon:osd"]
-# - ["ceph-mds:ceph-mds", "ceph-mon:mds"]
- # # Manila
- # - [ manila, keystone ]
- # - [ manila, manila-generic ]
- # - [ neutron-openvswitch, manila ]
diff --git a/bundle.yaml b/bundle.yaml
index 7c45d43156d8a7cb94396787257b578b238cb905..0811a44843f378f3f31b0f88abebf96595ec28d0 100644
--- a/bundle.yaml
+++ b/bundle.yaml
@@ -5,7 +5,7 @@
##
series: bionic
-description: A four-machine Kubernetes cluster, appropriate for production. Includes a three-machine etcd and ceph-mon cluster, a Kubernetes master, three Kubernetes worker nodes and a Kubernetes worker with GPUs. The Ceph (Luminous) cluster uses three nodes for Ceph Monitors and a Ceph OSD in LXD containers on each worker machine. Two Ceph MDSs provide redundant shared Ceph File System.
+description: A 9-machine Kubernetes cluster, appropriate for production. Includes a three-machine etcd and ceph-mon cluster, two Kubernetes masters, three Kubernetes worker nodes and a Kubernetes worker with GPUs. The Ceph (Luminous) cluster uses three nodes for Ceph Monitors and a Ceph OSD in LXD containers on each worker machine. Two Ceph MDSs provide redundant shared Ceph File System.
# ----------------------------------------------------------------------
machines:
@@ -30,17 +30,24 @@ machines:
# ----------------------------------------------------------------------
services:
-# defaultgw:
-# charm: cs:~csd-garr/defaultgw
-# options:
-# # ip route | grep default
-# gateway: 90.147.159.1/25
+# canal:
+# annotations:
+# gui-x: '450'
+# gui-y: '750'
+# charm: cs:~containers/canal
+# resources:
+# calico: 73
+# calico-arm64: 69
+# flannel: 73
+# flannel-arm64: 72
easyrsa:
charm: cs:~containers/easyrsa
bindings:
"": space-os-mgmt
num_units: 1
+ resources:
+ easyrsa: 5
to:
# an lxd on the unit of kubernetes-master
- "lxd:kubernetes-master"
@@ -56,6 +63,9 @@ services:
# constraints: tags=kubernetes
options:
channel: 3.2/stable
+ resources:
+ etcd: 3
+ snapshot: 0
to:
- "lxd:0"
- "lxd:1"
@@ -81,6 +91,7 @@ services:
apiserver: space-k8s-pub
expose: true
num_units: 1
+ resources: {}
to:
- "lxd:kubernetes-master"
annotations:
@@ -95,19 +106,26 @@ services:
kubernetes-master:
charm: cs:~containers/kubernetes-master
+ constraints: tags=kubernetes-master
bindings:
"": space-os-mgmt
#kube-api-endpoint: space-pub
num_units: 1
- constraints: tags=kubernetes-master cores=2 mem=4G #root-disk=16G
options:
- channel: 1.11/stable
- authorization-mode: "Node,RBAC,Webhook"
+ channel: 1.13/stable
+ authorization-mode: "Node,RBAC,WebHook"
# RBAC to allow access to dashboard to User "system:anonymous"
# Because of this bug https://github.com/kubernetes/kubernetes/issues/39722
# the value must be false, rather than what the documentation says.
api-extra-args: "anonymous-auth=false"
enable-dashboard-addons: True
+ resources:
+ cdk-addons: 0
+ kube-apiserver: 0
+ kube-controller-manager: 0
+ kube-proxy: 0
+ kube-scheduler: 0
+ kubectl: 0
annotations:
gui-x: '800'
gui-y: '850'
@@ -116,13 +134,13 @@ services:
kubernetes-worker:
charm: cs:~containers/kubernetes-worker
+ constraints: tags=kubernetes cores=4 mem=4G root-disk=16G
bindings:
"": space-os-mgmt
- num_units: 3
- constraints: tags=kubernetes-worker cores=4 mem=4G #root-disk=16G
expose: true
+ num_units: 3
options:
- channel: 1.11/stable
+ channel: 1.13/stable
# Enable Cloud Controller Manager
# https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/
# kubelet-extra-args: --cloud-provider=external
@@ -134,40 +152,16 @@ services:
- "2"
- "3"
-
-
-
-# ceph-proxy:
-# charm: "cs:ceph-proxy"
-# num_units: 1
-# bindings:
-# '': *network_space_default
-# options:
-## admin-key: AQAYOl5Zy5oPABAAGm3E5rQ11CWEs7etRU1Tqg==
-## fsid: "a5256fb9-3bba-4c19-bc1b-2e34fc132706"
-## monitor-hosts: "10.2.3.23:6789,10.2.3.22:6789,10.2.3.21:6789"
-# admin-key: AQAHQ4RVLRCZGxAAawt6yF14NEnSiZKk2MfyBA==
-# fsid: "9a9eedd0-9400-488e-96de-c349fffad7c4"
-# monitor-hosts: "10.2.3.243:6789,10.2.3.242:6789,10.2.3.241:6789"
-# annotations:
-# gui-x: "1300"
-# gui-y: "0"
-# to:
-# - "lxd:0"
-
-
-
kubernetes-worker-gpu:
charm: cs:~containers/kubernetes-worker
bindings:
"": space-os-mgmt
- # charm: ./kubernetes-worker # patched for cuda-9.0
num_units: 1
- constraints: tags=gpu cores=4 mem=4G #root-disk=16G
+ constraints: tags=gpu
expose: true
options:
- channel: 1.11/stable
- cuda_repo: 10.0.130-1
+ channel: 1.13/stable
+ # cuda_repo: 10.0.130-1
# kubelet-extra-args: --cloud-provider=external
annotations:
gui-x: "520"
@@ -175,83 +169,16 @@ services:
to:
- "4"
-# ceph-mon:
-# charm: cs:ceph-mon
-# num_units: 3
-# bindings:
-# "": space-os-mgmt
-# options:
-# # source: cloud:xenial-pike
-# to:
-# - 'lxd:0' # lxd:etcd
-# - 'lxd:1'
-# - 'lxd:2'
-# annotations:
-# gui-x: '1200'
-# gui-y: '400'
-
-# ceph-osd:
-# charm: cs:~csd-garr/ceph-osd
-# num_units: 3
-# bindings:
-# "": space-os-mgmt
-# cluster: space-os-data
-# options:
-# osd-devices: /dev/mapper/mpatha /dev/mapper/mpathb /dev/mapper/mpathc /dev/mapper/mpathd
-# osd-max-backfills: 1
-# osd-recovery-max-active: 1
-# osd-journal-size: 10000
-# use-syslog: True
-# # source: cloud:xenial-pike
-# # bluestore: True
-# to:
-# - "kubernetes-worker/0"
-# - "kubernetes-worker/1"
-# - "kubernetes-worker/2"
-# annotations:
-# gui-x: '1400'
-# gui-y: '400'
-
-# ceph-mds:
-# charm: cs:ceph-fs
-# num_units: 1
-# bindings:
-# "": space-os-mgmt
-# to:
-# - 'lxd:1'
-# annotations:
-# gui-x: '1200'
-# gui-y: '400'
-
- # manila:
- # charm: cs:manila
- # bindings:
- # "": space-os-mgmt
- # to:
- # - 'lxd:2'
- # options:
- # debug: True
-
- # manila-generic:
- # charm: cs:manila-generic
- # bindings:
- # "": space-os-mgmt
- # to:
- # - 'lxd:2'
- # options:
- # debug: True
# ----------------------------------------------------------------------
relations:
- ["kubernetes-master:kube-api-endpoint", "kubeapi-load-balancer:apiserver"]
- ["kubernetes-master:loadbalancer", "kubeapi-load-balancer:loadbalancer"]
- ["kubernetes-master:certificates", "easyrsa:client"]
-# - ["kubernetes-master:juju-info", "defaultgw:juju-info"]
- ["kubernetes-master:juju-info", "kubernetes-keystone:juju-info"]
- ["kubernetes-master:etcd", "etcd:db"]
# Kube load balancer
- ["kubeapi-load-balancer:certificates", "easyrsa:client"]
-# - ["kubeapi-load-balancer:juju-info", "defaultgw:juju-info"]
- ["etcd:certificates", "easyrsa:client"]
# Kube worker
- ["kubernetes-master:kube-control", "kubernetes-worker:kube-control"]
@@ -261,19 +188,14 @@ relations:
- ["kubernetes-master:kube-control", "kubernetes-worker-gpu:kube-control"]
- ["kubernetes-worker-gpu:certificates", "easyrsa:client"]
- ["kubernetes-worker-gpu:kube-api-endpoint", "kubeapi-load-balancer:website"]
+ # Canal
+# - ["canal:etcd", "etcd:db"]
+# - ["canal:cni", "kubernetes-master:cni"]
+# - ["canal:cni", "kubernetes-worker:cni"]
+# - ["canal:cni", "kubernetes-worker-gpu:cni"]
# Flannel
- ["flannel:etcd", "etcd:db"]
- ["flannel:cni", "kubernetes-master:cni"]
- ["flannel:cni", "kubernetes-worker:cni"]
- ["flannel:cni", "kubernetes-worker-gpu:cni"]
- # Ceph
-# - ["kubernetes-master:ceph-storage", "ceph-mon:admin"]
-# - ["kubernetes-master:ceph-storage", "ceph-proxy:client"]
-
-# - ["ceph-osd:mon", "ceph-mon:osd"]
-# - ["ceph-mds:ceph-mds", "ceph-mon:mds"]
- # # Manila
- # - [ manila, keystone ]
- # - [ manila, manila-generic ]
- # - [ neutron-openvswitch, manila ]