Merge pull request #3854 from consideRatio/pr/linc

linc: add cluster, and deploy support chart and staging hub

.github/workflows/deploy-grafana-dashboards.yaml

@@ -23,6 +23,7 @@ jobs:
           - cluster_name: hhmi
           - cluster_name: jupyter-meets-the-earth
           - cluster_name: leap
+          - cluster_name: linc
           - cluster_name: linked-earth
           - cluster_name: meom-ige
           - cluster_name: nasa-cryo

.github/workflows/deploy-hubs.yaml

@@ -178,32 +178,33 @@ jobs:
     #
     # If you are adding a new cluster, please remember to list it here!
     outputs:
-      failure_2i2c: "${{ env.failure_2i2c }}"
+      failure_2i2c-aws-us: "${{ env.failure_2i2c-aws-us }}"
       failure_2i2c-uk: "${{ env.failure_2i2c-uk }}"
+      failure_2i2c: "${{ env.failure_2i2c }}"
+      failure_awi-ciroh: "${{ env.failure_awi-ciroh }}"
+      failure_catalystproject-africa: "${{ env.failure_catalystproject-africa }}"
+      failure_catalystproject-latam: "${{ env.failure_catalystproject-latam }}"
       failure_cloudbank: "${{ env.failure_cloudbank }}"
+      failure_earthscope: "${{ env.failure_earthscope }}"
+      failure_gridsst: "${{ env.failure_gridsst }}"
+      failure_hhmi: "${{ env.failure_hhmi }}"
+      failure_jupyter-meets-the-earth: "${{ env.failure_jupyter-meets-the-earth }}"
       failure_leap: "${{ env.failure_leap }}"
-      failure_meom-ige: "${{ env.failure_meom-ige }}"
-      failure_openscapes: "${{ env.failure_openscapes }}"
-      failure_pangeo-hubs: "${{ env.failure_pangeo-hubs }}"
-      failure_utoronto: "${{ env.failure_utoronto }}"
+      failure_linc: "${{ env.failure_linc }}"
       failure_linked-earth: "${{ env.failure_linked-earth }}"
-      failure_awi-ciroh: "${{ env.failure_awi-ciroh }}"
+      failure_meom-ige: "${{ env.failure_meom-ige }}"
       failure_nasa-cryo: "${{ env.failure_nasa-cryo }}"
-      failure_gridsst: "${{ env.failure_gridsst }}"
-      failure_victor: "${{ env.failure_victor }}"
-      failure_2i2c-aws-us: "${{ env.failure_2i2c-aws-us }}"
-      failure_ubc-eoas: "${{ env.failure_ubc-eoas }}"
-      failure_nasa-veda: "${{ env.failure_nasa-veda }}"
+      failure_nasa-esdis: "${{ env.failure_nasa-esdis }}"
       failure_nasa-ghg: "${{ env.failure_nasa-ghg }}"
+      failure_nasa-veda: "${{ env.failure_nasa-veda }}"
+      failure_openscapes: "${{ env.failure_openscapes }}"
+      failure_opensci: "${{ env.failure_opensci }}"
+      failure_pangeo-hubs: "${{ env.failure_pangeo-hubs }}"
       failure_qcl: "${{ env.failure_qcl }}"
-      failure_jupyter-meets-the-earth: "${{ env.failure_jupyter-meets-the-earth }}"
       failure_smithsonian: "${{ env.failure_smithsonian }}"
-      failure_catalystproject-latam: "${{ env.failure_catalystproject-latam }}"
-      failure_catalystproject-africa: "${{ env.failure_catalystproject-africa }}"
-      failure_hhmi: "${{ env.failure_hhmi }}"
-      failure_nasa-esdis: "${{ env.failure_nasa-esdis }}"
-      failure_earthscope: "${{ env.failure_earthscope }}"
-      failure_opensci: "${{ env.failure_opensci }}"
+      failure_ubc-eoas: "${{ env.failure_ubc-eoas }}"
+      failure_utoronto: "${{ env.failure_utoronto }}"
+      failure_victor: "${{ env.failure_victor }}"
 
     # Only run this job on pushes to the default branch and when the job output is not
     # an empty list

config/clusters/linc/cluster.yaml

@@ -0,0 +1,20 @@
+name: linc
+provider: aws # https://2i2c.awsapps.com/start#/
+aws:
+  key: enc-deployer-credentials.secret.json
+  clusterType: eks
+  clusterName: linc
+  region: us-east-1
+support:
+  helm_chart_values_files:
+    - support.values.yaml
+    - enc-support.secret.values.yaml
+hubs:
+  - name: staging
+    display_name: MIT Linc Staging
+    domain: staging.linc.2i2c.cloud
+    helm_chart: daskhub
+    helm_chart_values_files:
+      - common.values.yaml
+      - staging.values.yaml
+      - enc-staging.secret.values.yaml

config/clusters/linc/common.values.yaml

@@ -0,0 +1,53 @@
+basehub:
+  nfs:
+    enabled: true
+    pv:
+      enabled: true
+      # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html
+      mountOptions:
+        - rsize=1048576
+        - wsize=1048576
+        - timeo=600
+        - soft # We pick soft over hard, so NFS lockups don't lead to hung processes
+        - retrans=2
+        - noresvport
+      serverIP: fs-0276405f3cabae08b.efs.us-east-1.amazonaws.com
+      baseShareName: /
+  jupyterhub:
+    hub:
+      config:
+        JupyterHub:
+          authenticator_class: github
+        GitHubOAuthenticator:
+          populate_teams_in_auth_state: true
+          # allowed_organizations:
+          #   - abc:def
+          scope:
+            - read:org
+        Authenticator:
+          enable_auth_state: true
+          # admin_users:
+          #   - asdf
+    custom:
+      2i2c:
+        add_staff_user_ids_to_admin_users: true
+        add_staff_user_ids_of_type: "github"
+      jupyterhubConfigurator:
+        enabled: false
+      homepage:
+        templateVars:
+          org:
+            logo_url: ""
+            url: ""
+          designed_by:
+            name: 2i2c
+            url: https://2i2c.org
+          operated_by:
+            name: 2i2c
+            url: https://2i2c.org
+          funded_by:
+            name: ""
+            url: ""
+    scheduling:
+      userScheduler:
+        enabled: true

config/clusters/linc/enc-deployer-credentials.secret.json

@@ -0,0 +1,25 @@
+{
+	"AccessKey": {
+		"AccessKeyId": "ENC[AES256_GCM,data:obOoEAg96TUdY+rBDp2nWpGiQ4E=,iv:dVtyMB/d2usnEiOgsQEDDVGuM2Di0qzez2RT2qnt01E=,tag:6ot0Os/64E89A7zPFILljA==,type:str]",
+		"SecretAccessKey": "ENC[AES256_GCM,data:XL8EYVl1ntBn2lF8+nPvxj0LUQMfXZWmRTABAXYQ0ez3IOiJn4Wcjw==,iv:uKnfzEmcpX0tIuH4JhMRDumWNFHMEphc8gsYapbWI1w=,tag:PLb9a0mGmPpqV9+ytohfSw==,type:str]",
+		"UserName": "ENC[AES256_GCM,data:DlTmkL/Osn+TFxu0ddV9CC0sAvjDj9g=,iv:mBMJNleUt6s3W1ZG96jeEgx1/8gbTL+EuwzAxRxK840=,tag:ZXZt9gAUQqhXfwdppKjPOQ==,type:str]"
+	},
+	"sops": {
+		"kms": null,
+		"gcp_kms": [
+			{
+				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
+				"created_at": "2024-03-26T15:19:58Z",
+				"enc": "CiUA4OM7eNHxko8FmJx7qgQUbKHY3ABCe8rJErxdTt6tKMOp4L0uEkkAXoW3Jh1L5kIsyg7ix0MdFQj/wNuAzinGsGTbMVmFcX7w/+Pwoqx3clgp2oG9D4jeSfDkqd49poH2LF7fN6uvd/zHcwyTBVXA"
+			}
+		],
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2024-03-26T15:19:59Z",
+		"mac": "ENC[AES256_GCM,data:8XdKuQnDYS4a3eFh0gmYTZI5BLl5m3PSnO8iqLH0CFHA22cK74uJLV5hMmew31wsj49AiegfWYN0FvCZIgMpgI8K+KGDed1+2JekMxsw+0eNEGcAfe2J+tNMw4vRGtLflQQ69Ti3n5lvxnH4hu803KRLzpCkIZOQXj2Mni24IAQ=,iv:2dW0TocUokRTovj1MbCQKuQ6FiBWJGqclztqIOYG4NQ=,tag:2GDKtyRN+UTCOeT1TbLMTQ==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.8.1"
+	}
+}

config/clusters/linc/enc-grafana-token.secret.yaml

@@ -0,0 +1,15 @@
+grafana_token: ENC[AES256_GCM,data:kIEiFXS/cN5tKDGew4Wl5tng9r5z0yPu453jtzcVCqF9dY4OuCEajsGMCMhJYQ==,iv:IrLuXFo1iWRwZli/wGaBqlN2fP6ZO/u9Co1+4OZUVQ8=,tag:6K6ZBmY/JuYfmIO2Jibxcg==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+          created_at: "2024-03-26T19:06:56Z"
+          enc: CiUA4OM7eHCt0w2TUePPc5KRUvBsuLMUbxto1N1qOpqdK+Kdd+aQEkkAXoW3JkGGOlVo/b3ye3FSYzNh9AexBXjEHVEBKfG58kQvAGFBO8Lm+ZPzijCFCwWvKC8iLLFKuRjSZQ9vlYk0CnICNzj1fJdE
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-03-26T19:06:56Z"
+    mac: ENC[AES256_GCM,data:q6V5FUfmO2Tp8viPS7sWccyw3giZewA+y51pvQNIctQ2s/tXfbFWxrznF4/Tne77daMa/u7DGIZZDoNQKHKjfcsg7drP7D5GTJORcbLtYIXvhOW92Z8Wmo0kl6+is+lW3UfyVo7nUtZsgsNXay5kd2/ajc1KXGBO/7BFowBcar0=,iv:oCjf/KA5+zwrbuVPLno6QG1fZcxQy/B3s4XuGC0H7MM=,tag:vc1tuC+VJ/XaEPRDFJoGCQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

config/clusters/linc/enc-staging.secret.values.yaml

@@ -0,0 +1,21 @@
+basehub:
+    jupyterhub:
+        hub:
+            config:
+                GitHubOAuthenticator:
+                    client_id: ENC[AES256_GCM,data:iqSeYxc7IMNhcpNGiRmeWOl094k=,iv:1FZm7sWuuXMQeO58nZbwa/JwDlzA8VlJNfe9ch6LcKI=,tag:5qLvu3c/cMTPtPFt0+KSTw==,type:str]
+                    client_secret: ENC[AES256_GCM,data:xGqTAceOHIXAUV1T6L3l88XHcz+CxfmThoyBqCruFlzWoPmx7fGOGg==,iv:FLevQvMe+JuAi3uxhItGlgjj6jr9sU+0NJHuwlCwQi8=,tag:VNkTtaHvtX+/kj9D3fI1DA==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+          created_at: "2024-03-26T20:12:17Z"
+          enc: CiUA4OM7eNNn06TvKdpGzzSeNZuhYETgaxlOvCcPguaIfbTpp6pvEkkAXoW3JkNqEFls9o/uPAImn37WfgEq77tvHn2/XPm1Es4zGVKRF0izBUlKS1tTDP5307XZFnapSoMiPr2ICOgBv7/KB7bVTDLz
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-03-26T20:12:17Z"
+    mac: ENC[AES256_GCM,data:6PnxDsjI2xpBBs5PtktXO2rCOj1Uml/C0xp7IMClXAK8scxGtcQ8XnntieeiCHtAon0yY5IQP+swDwY1ZpixnL34O/B0JLX86mANM7lhos4DFeC2RtAxeYZvdgz0yWOU5k4A2So/dhEWpPK19AlplZ9YXbTDdZR6N50fjhg0PUw=,iv:7/5Qw1Wj38FMvVxHEU+Jvoie/jeX2qmPuXwaPm4qlxU=,tag:ykDpQFuaI/sZC4O2HvXkJw==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

config/clusters/linc/enc-support.secret.values.yaml

@@ -0,0 +1,22 @@
+prometheusIngressAuthSecret:
+    username: ENC[AES256_GCM,data:KnFsl1f9JzCmlkO9lRsD7WlgVl7QaAKw6yRARWqXAr922ZdQBjFuP0U9RV3mOqywtN6BAghW7xWk1hJPJsGPpQ==,iv:wbPi9YqYibvUXvNpA0Zqgocw2APKv3+sSm+ulLAxqQw=,tag:nwAVKHo2b+vGB2d9LhAyyg==,type:str]
+    password: ENC[AES256_GCM,data:u/0+dh7WOm4XwRuMIeHog3csEj3D6+vExyboJZJ/kGb9OpAhNzECSjm6Jk0aJcQWCq7Bfur7hf7nAvMzOcTbQw==,iv:J7c7OVZIjQXC2FOHEtMhX4SNhmh9aolKOgsAccu3iAo=,tag:iKXUJEOa9/kNt6szLLZfnw==,type:str]
+grafana:
+    grafana.ini:
+        auth.github:
+            client_id: ENC[AES256_GCM,data:eAfKvoZDnyAskxeDA89WrTa/E64=,iv:tXullNgwUB4dBoaYzzgg34HlkGX6VEFkZMwZ5v677Cw=,tag:7MULePUMAhxkJ+4jdWxMxA==,type:str]
+            client_secret: ENC[AES256_GCM,data:bm9lHXiXI69hWz03gMwJsCbkWMhHpHyJK/DMub8BnmbWXILIBJbCDA==,iv:42IjM17W/BPQMBX4w336NRXhtc/RnUegbpCeHJxU84Y=,tag:7DyZQym+d01tmErPqDjvdg==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+          created_at: "2024-03-26T14:01:51Z"
+          enc: CiUA4OM7eFnE/G6kcDKgG8VNSZ8l+2EzjyWaKoGzi7oP7dudbOy4EkkAXoW3JgUyIqJjkz37SB2zkVaJsQMrfq8wR27oiAyuULvsc10K6xTQBfIabCXk/uILwPwkoo14Mw9oyfbZyZyfvgUwzEg/pCSB
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-03-26T19:51:46Z"
+    mac: ENC[AES256_GCM,data:xVT9dTtzgw72XnoSsA8oU3WrFcLLucVnUPiPrA3uySQygmfmBK1E6+oh4SUWccbiR+CgBbj/Qa2ppbwYikokjpUDOp+5JumQab105amCOoevMAZBgq1VoTlhqAuITQR1VirikMlIf5gSnl44DcF0AHHr0C5Z9Hgz7ig2a8X8Hss=,iv:AUrNWiT06Sk3LV/u6FJg5o59GgzSDDNkLr+4jUxpJIg=,tag:dm9MF1ulF0R3D1X8qz8RxA==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

config/clusters/linc/staging.values.yaml

@@ -0,0 +1,16 @@
+basehub:
+  jupyterhub:
+    ingress:
+      hosts: [staging.linc.2i2c.cloud]
+      tls:
+        - hosts: [staging.linc.2i2c.cloud]
+          secretName: https-auto-tls
+    hub:
+      config:
+        GitHubOAuthenticator:
+          oauth_callback_url: https://staging.linc.2i2c.cloud/hub/oauth_callback
+    custom:
+      homepage:
+        templateVars:
+          org:
+            name: MIT Linc (staging)

config/clusters/linc/support.values.yaml

@@ -0,0 +1,34 @@
+prometheusIngressAuthSecret:
+  enabled: true
+
+prometheus:
+  server:
+    ingress:
+      enabled: true
+      hosts:
+        - prometheus.linc.2i2c.cloud
+      tls:
+        - secretName: prometheus-tls
+          hosts:
+            - prometheus.linc.2i2c.cloud
+
+grafana:
+  grafana.ini:
+    server:
+      root_url: https://grafana.linc.2i2c.cloud/
+  auth.github:
+    enabled: true
+    allowed_organizations: 2i2c-org
+  ingress:
+    hosts:
+      - grafana.linc.2i2c.cloud
+    tls:
+      - secretName: grafana-tls
+        hosts:
+          - grafana.linc.2i2c.cloud
+
+cluster-autoscaler:
+  enabled: true
+  autoDiscovery:
+    clusterName: linc
+  awsRegion: us-east-1

config/clusters/templates/common/support.values.yaml

@@ -1,6 +1,6 @@
 prometheusIngressAuthSecret:
   enabled: true
-  
+
 prometheus:
   server:
     ingress: