[New] Opensci cluster and hub

config/clusters/opensci/cluster.yaml

@@ -0,0 +1,8 @@
+name: opensci
+provider: aws
+aws:
+  key: enc-deployer-credentials.secret.json
+  clusterType: eks
+  clusterName: opensci
+  region: us-west-2
+hubs: []

config/clusters/opensci/enc-deployer-credentials.secret.json

@@ -0,0 +1,25 @@
+{
+	"AccessKey": {
+		"AccessKeyId": "ENC[AES256_GCM,data:MtyZwyAG9hUN2TZmVBY99AUkTzk=,iv:X1yxWvoAR4qlzPGDr9sh5fI5/nPqsKezibr/gJ6sGyI=,tag:JkExYO+KJxqrBep71B+tpw==,type:str]",
+		"SecretAccessKey": "ENC[AES256_GCM,data:k5ZOOtSBK6GQG60fkcuVju/zuIzyXSmou+lMpbqI9KXj/70nK2vMxw==,iv:rxPpG9bTHAFB6TbtZoJQ6CglXHnDk0d6+3OV3//TqUs=,tag:CksFoyD6jh7Bd3tIZHQvug==,type:str]",
+		"UserName": "ENC[AES256_GCM,data:POvIw42gLg8qNOAQeZsvyi+Zma/I5Jo=,iv:uMiKk7ONZxSMm5K/rSEgOL1ZusHy8VgFD9C2D2ezEcg=,tag:oDJTF8RmGwpCBpEjvqL+PA==,type:str]"
+	},
+	"sops": {
+		"kms": null,
+		"gcp_kms": [
+			{
+				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
+				"created_at": "2024-02-21T15:39:23Z",
+				"enc": "CiUA4OM7eF5o6mB9Vayi+puvS7aVXCANRtsaycfD68b7ISp9B6drEkkAXoW3JtPtnpYszaNYGfUeJiDVthqBYPcRJjtmCPqm6DEVL9Uyyordh2F636IlremL8X5LedANy3V6JQfofNHug3SiOYSzTqaj"
+			}
+		],
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2024-02-21T15:39:24Z",
+		"mac": "ENC[AES256_GCM,data:mX6G6KmXOkBiUMT/robJTZ2L8KozL2S8av0UIBhO7lNWo4BJJYLNx9fQL6wzjgWkclE8NI6AiZg57qo6u8SCIV+Fg1veJjsTv9mxOtuV1NbSH8vLs8FOCq0Qp/qDUTFCTIATqqIGPaTB6oUeM7TkBAlwS3SedRn/GTVMAFFDjbY=,iv:zncrRM8g/aC+oh/Hoogil+kUSst/GXOrQbKOwtbw1G4=,tag:V4wlwuxyDLVdLn9t1No41Q==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.3"
+	}
+}

config/clusters/opensci/enc-support.secret.values.yaml

@@ -0,0 +1,17 @@
+prometheusIngressAuthSecret:
+    username: ENC[AES256_GCM,data:NAA8fg7Oin4CLlFAR0/q9I0FpqHHsyXntce7by5Fg4B4PVGnmboc6hiHKbcvq4gkhFu3JkSPO/UZOnAi/vPXVA==,iv:t21nYjrvFgJ5vRM/8FDGwMrlGiLYsE9R4+BFxjDf91c=,tag:gnyjHQCSfouHljf6AzQKiw==,type:str]
+    password: ENC[AES256_GCM,data:Dcu0hyudGn0a51p8yutj2MbMv0ydSS/ewXqDF1xAVsWV75DUikNjnqxKZWbBDmjZisi+lMiRHZEUrxaszcGE9w==,iv:AM/9clOgMS80/JdZb1UC9fZNliQwhD8BJdZmSk7+Xow=,tag:kV4UPf5vPkTjESJGNusS9A==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs
+          created_at: "2024-02-21T14:04:13Z"
+          enc: CiUA4OM7eH9GfolTeTic397lI94/FljLr1s7Hz77OOck8EsW/8pvEkkAXoW3JqTtm0UrLSlLBrebh+OQ+6ik5KFXmY8Xxl9ICv9kSnbz7CFBvAHlhrP7W7/NK8ZP5+6NnOivp0SZlghOW9M5Lv5ZpnQc
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-02-21T14:04:13Z"
+    mac: ENC[AES256_GCM,data:IlvuWpEYx2Qjp12hXHSnQdS9RYU1lwH2L8CgE1Js2cXRzhFr+cRalpJ68h/G8uzJOowb/WI5svSBB372HoX0FSf3kRmUPBdj0nI0Leb7kzZoOWJfVsCNh+Z7KVqs7iBnCWRtIr5v00eD6WUf1Q93qgxgcuZgAewd8rzaiixN0GE=,iv:I6/qm0v3/kBt+zFXm/jM29wo3ZW8p6xT9cfI+ruJGCQ=,tag:F2rZt0kHoHSsdECq0IY7eQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3

config/clusters/opensci/support.values.yaml

@@ -0,0 +1,28 @@
+prometheusIngressAuthSecret:
+  enabled: true
+
+prometheus:
+  server:
+    ingress:
+      enabled: true
+      hosts:
+        - prometheus.opensci.2i2c.cloud
+      tls:
+        - secretName: prometheus-tls
+          hosts:
+            - prometheus.opensci.2i2c.cloud
+
+grafana:
+  grafana.ini:
+    server:
+      root_url: https://grafana.opensci.2i2c.cloud/
+  auth.github:
+    enabled: true
+    allowed_organizations: 2i2c-org
+  ingress:
+    hosts:
+      - grafana.opensci.2i2c.cloud
+    tls:
+      - secretName: grafana-tls
+        hosts:
+          - grafana.opensci.2i2c.cloud

docs/topic/access-creds/cloud-auth.md

@@ -102,11 +102,12 @@ To do so, follow these steps:
    after logging in for current set of IAM users.
 2. Go to the [SSO users](https://console.aws.amazon.com/singlesignon/identity/home?region=us-east-1#!/users)
    page, and create an appropriate entry for the new user.
-   a. Their username should match their `2i2c.org` email address.
-   b. Use their `2i2c.org` address as email address.
-   c. Other than email and username, provide as little info as possible. This would be
+
+   - Their username should match their `2i2c.org` email address.
+   - Use their `2i2c.org` address as email address.
+   - Other than email and username, provide as little info as possible. This would be
       just first name, last name and display name.
-   d. "Send an email to the user with password setup instructions".
+   - "Send an email to the user with password setup instructions".
 3. Add them to the `2i2c-engineers` group. This gives them access to all the other
    AWS accounts we create.
 4. Create the account! They'll receive an email with appropriate instructions.

eksctl/opensci.jsonnet

@@ -0,0 +1,128 @@
+/*
+    This file is a jsonnet template of a eksctl's cluster configuration file,
+    that is used with the eksctl CLI to both update and initialize an AWS EKS
+    based cluster.
+
+    This file has in turn been generated from eksctl/template.jsonnet which is
+    relevant to compare with for changes over time.
+
+    To use jsonnet to generate an eksctl configuration file from this, do:
+
+        jsonnet opensci.jsonnet > opensci.eksctl.yaml
+
+    References:
+    - https://eksctl.io/usage/schema/
+*/
+local ng = import "./libsonnet/nodegroup.jsonnet";
+
+// place all cluster nodes here
+local clusterRegion = "us-west-2";
+local masterAzs = ["us-west-2a", "us-west-2b", "us-west-2c"];
+local nodeAz = "us-west-2a";
+
+// Node definitions for notebook nodes. Config here is merged
+// with our notebook node definition.
+// A `node.kubernetes.io/instance-type label is added, so pods
+// can request a particular kind of node with a nodeSelector
+local notebookNodes = [
+    { instanceType: "r5.xlarge" },
+    { instanceType: "r5.4xlarge" },
+    { instanceType: "r5.16xlarge" },
+];
+local daskNodes = [];
+
+
+{
+    apiVersion: 'eksctl.io/v1alpha5',
+    kind: 'ClusterConfig',
+    metadata+: {
+        name: "opensci",
+        region: clusterRegion,
+        version: "1.28",
+    },
+    availabilityZones: masterAzs,
+    iam: {
+        withOIDC: true,
+    },
+    // If you add an addon to this config, run the create addon command.
+    //
+    //    eksctl create addon --config-file=opensci.eksctl.yaml
+    //
+    addons: [
+        {
+            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+            // couple to AWS EBS based storage, without it expect to see pods
+            // mounting a PVC failing to schedule and PVC resources that are
+            // unbound.
+            //
+            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+            //
+            name: 'aws-ebs-csi-driver',
+            version: "latest",
+            wellKnownPolicies: {
+                ebsCSIController: true,
+            },
+        },
+    ],
+    nodeGroups: [
+        ng + {
+            namePrefix: 'core',
+            nameSuffix: 'a',
+            nameIncludeInstanceType: false,
+            availabilityZones: [nodeAz],
+            ssh: {
+                publicKeyPath: 'ssh-keys/opensci.key.pub'
+            },
+            instanceType: "r5.xlarge",
+            minSize: 1,
+            maxSize: 6,
+            labels+: {
+                "hub.jupyter.org/node-purpose": "core",
+                "k8s.dask.org/node-purpose": "core"
+            },
+        },
+    ] + [
+        ng + {
+            namePrefix: 'nb',
+            availabilityZones: [nodeAz],
+            minSize: 0,
+            maxSize: 500,
+            instanceType: n.instanceType,
+            ssh: {
+                publicKeyPath: 'ssh-keys/opensci.key.pub'
+            },
+            labels+: {
+                "hub.jupyter.org/node-purpose": "user",
+                "k8s.dask.org/node-purpose": "scheduler"
+            },
+            taints+: {
+                "hub.jupyter.org_dedicated": "user:NoSchedule",
+                "hub.jupyter.org/dedicated": "user:NoSchedule"
+            },
+        } + n for n in notebookNodes
+    ] + ( if daskNodes != null then
+        [
+        ng + {
+            namePrefix: 'dask',
+            availabilityZones: [nodeAz],
+            minSize: 0,
+            maxSize: 500,
+            ssh: {
+                publicKeyPath: 'ssh-keys/opensci.key.pub'
+            },
+            labels+: {
+                "k8s.dask.org/node-purpose": "worker"
+            },
+            taints+: {
+                "k8s.dask.org_dedicated" : "worker:NoSchedule",
+                "k8s.dask.org/dedicated" : "worker:NoSchedule"
+            },
+            instancesDistribution+: {
+                onDemandBaseCapacity: 0,
+                onDemandPercentageAboveBaseCapacity: 0,
+                spotAllocationStrategy: "capacity-optimized",
+            },
+        } + n for n in daskNodes
+        ] else []
+    )
+}

eksctl/ssh-keys/opensci.key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCronsz3J+j6Lbp8BJ/mpTn25/1dgmpZQpoLjEqflTtK0Ur/JBZ+P05aD6eLKy89ANyD9RC/x1oHgDwIAtN7zoCG6QJA+8dWUhqlESYzfQ4ieSW36oi1k4VxPLVUU4zVMPr71tdBaFPOsE3jTEWJxSeDZ8YjzHa0sr5X8HzuACO0V5HG8FnSxsXCPAAtwR1VdZF9xRjhQ8u4yLhDz/kTb1vXCjgz0CcHIs7b7ZSxY/tLxIeCTjfyw3SSvHPTZRgZgoqY5mXwgxq4mLQ96u19u6T9zW1qTjHqVQ2OhuAfIC7LzKtZq2S+RlPiv9xoP1+vK2OULZMtdH/u8VHXGGws4iiPVo+xuADtIKAVZCXgrkvHP//bN4pFObzxsxWa2w1m+mGcXKLmHWy3sqSYZHIJwYhcXQXSSiZF56fFKKtCoT/0Ddk6FdMr9WqNjc3Tq12AfXQgobn0JbDUhYRl5bLVfIo7urshDEfTN+fRvcMnDKCNEmnT/ihn5Ki9BPP+Q8NTOU= [email protected]

eksctl/ssh-keys/secret/opensci.key

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:3U6QvkVPU64GcF+6QaYPnPff9ILQLpmRYlL7yWVZbDGP5z/YTwoIRh+eIN+78zAmQwiJgWDpKN6Y3MAPYpmzKaktXzsz6tnADGBZlL28kYDHkgVd8iqexoQgq5JCqNKnw28rzMLlEpUMGefu8ZZMxpBcJu8DfvvymnV6hfjmPxCRSTBj+ZqDQtdDoMPNrZq9tF/W+Rs1yDIlImJpsy1KGsWQSluShr1t2wrfuZEcHZXaCg9qd9qRD1UTCq+mFhYy9oC8XKdHYOMrVNynau2pr3JOWhNBYxVGdEUEu13w0ySrBL9FWKhpoLetqIgtetm8V9Cvfd/DjbhuPPuoWN4Hpxr7o3/NR0MEdg4H6YI1IgyrYU6oF59032z8JZVFSktBwSSyPREyxTWhIuuqKCXoz7ZP9JSUt9aE8oJZD7GvRz4yOBi5/7Eyhg1Jof/CYqF7ncUdAJk1sprh2WV3FclpoDjaM2scgBAjy9HjTRUB0jKhQ/nOzVdVguTTvEPOqEbu1dx2D8p3JOQFpCHVmlBKpBmq35X1L0SqR4EkvcSFySqstVMt2vBmOdy2gx8Gl7HaSV1HBnDJWvFomESzI5nxB+ZFrtv+5ljYQP4o02fTCSa0fcEhsz64MpZHh8BlULQDh/FyUSUn+1otiNoLmd0tIhAMRcGslHvGzM1st9oyVP0dRAvG2cxZaRflpOpS63ezjHpCRmYgDuRcogNTTbrXoFSvZEhqamGmQTgNEte0GGtIU6U7NX2EMLloU+znqT7OQJyNHBByx2tcC4dxzc105P2VusWFPFEN4gL2ltq74TBGwIckeBC2b6BQuIgrBrc49z56HwEszWkpLEgHnjW1R+qCFxb4JEPlUtCTRvB+n3rdBKHHtvhLZhHM9LKNbBqr/10gwcBqD8i7B3bGwRvYeIKNOU5/UWtBxy2Byn9P+Z1RVTrXLNLXDjE/MfsWd3NInxgnQsPzpg/ZiESZH/cgD9KQmI+VM/tXbmaQXIyo4tod5x0zh50Y52/yqHZ7uKtPjetM/oJwRiKGbEQcMPqAdN9m5JKP9gyYn/sdIRPVmSoD9+5spxE96sBb1kwiuQXDdXFwVQsBZ1eGr2nRkLTddsX13+oPKVVPbYukGTRPDPZR7TyjQ+ztXTybsmQ3h8mbZ5Xf20UhSs63nkh0W691Gyo6H1d2lAJFBh5doKl0/i8v1im8kyip5GrH8xoVmCfEVYSRXbotXY4ccqzPoCqsh4VumeM3O4qa+fZLaVGsGc0UsR+9MWcmZ2E8HTEZi2vnDQA/wgEn6zbCfoxyaoX7nC/mPTcgRhvfPRRfZEuPIaTTGOaEyt85QpSVSqf5diZf/DaoYsIMasQ0XI/JH2fDcXtduc86kW9bdOS2RdG3XqMBJWdI5XmcdaDw5stQTxXFRsVy1jshAm6SXprMs54LbiQPXmzVEiOz8xjVlqmH40BN4ihbyW1VLujSQkUZ+wZ9cRhNYiSGNUUCBtajcdYSYOsLw+nvzv9DZB6wl8pBJkwElOuGaaiFrlpMdEJYQrkwkwIdIv6lx5oe9QH9SFrLbXS3i7+Vffh+mzwu/BLnQi4tRJJjtI3ORo94od3dZXoiDUJikl7zFXoiE2bgBvFhVDd2UTRwpbql3iW/dcvBzsHPtfiXsUKSPhGCGb5pVuTSEwXs0reK+V+/aY/BWuH7DFUEAvVqPSr0OpkK3y2Ok1XK3+4AdSWDPYo3nPy89b8Aid4Bj+BEb0nOeEl7uOTxR0D0emTgyCTffHV+eoYFMfsozmcME0y93GuEBe800GLaucmKjUHN8QTUpgXr1p2ReLgL3uayvnOQ4eEJQOEPYr4seKfDzDzCLabCd8o77hgnQReoDl6RmLpOiJd0KsFl8oUv03egIdQN+2++NYCd+H4n+S5rWE6wZWqwKU9ieADyZxTr2QGO22vFliktWVx09As+4R96NfCjoVLgY8pFBaC00HArGsPKVWp0zrPgBdjnup5J9JJOMK1dJKN/V0Ha4xbVuskDXY8JGZMeUc04lAg1s7ylm5w3vqnSPJ80w5wgYXfTy9t+fFTxSq//wqe0bgAM/gGgS25+JcSad84N4uaNvuj/nX0zWb1TUUXdKUt1sdptssgg5ExCryRMX0oVlUNhMD4jrpK1oyzIiUUWOUyawLpDWT29tgVBVFTVd4KiDuJThqWa1amAjYL+tsk6IStxNuFIzs3390JWg4WIOy0eg14Cr3J1JaND/IK97d3Y4IdewRhfR21TPZT8sRGg+V5oP0rMXxyte8YARaKRNY9j1X+zg4cEtg2mFw32UkxEpEHHSSk/1JPGJoCAo6AhdIalc9hb3gzqTlPC+roJvQT7KEftE+ZLyXu1lwIwXor0wYtpz2u6SNi+cHBRh3T2dYKkhrklKTMq2ZoRzh9xwoEL3cfzFeFv0XbbJtfILmSxt5KitRisv1w21g/tn746psfRHXD/FIpj5WDr21QCkW0tvsKvWc72Ww6Furnljay1osBHJ5GLgLQdZMNMF4t2nh3De2YMyYxfdprcZQCLtsIzBv1f+seqV3ffUTGDpNCPJryMAreZJS4+7X0qKEOR6jt/yyUPz6r7mYGMA5YBZ4wYuwo5SpQpLn5s1dVTseaLD+giSV6QHR8czs1ZohUMmqAXxBvX0wcCvsYcyFu5kd92zm0MIfZv2k5RHMHl4fwa936oPGd1pXkInR42K38PTrKAgJn5XcoyZ91ebUI4gcw4DaQX9D66T5Td8GLijOkI/reBIXhqLjDRvz9d3Odl21lnrOppBKGTA5kN3IODqxEE92Wqfexmaq6uIJUCA67PHgpuRK7Z/KxOYwjXXCgejazymicB7XYbMRSTxJWAHEo8pmuh/7TgZe+1YENHYnZ+QguR4guSuu6iVSRTtsUSA9ilgFkvoZhKJ4vDMBMeZ7FBPV8uix3C2vM8ukx5YnbKEmYLnoDQKS5mLxs3iUurtYw27+kmIemaKUjElTcdmXFNvOBqru6M0mWNZqW1DH+WeIHxQMjK1zQ8iyiA5K4+46ytNrHxkEEc66Pj55/x21O5wP3ETDigv2gOnX/dY3erFhP9N4/xlIoiYwYi/4P/IzQLd1tXwsd4iz+auoRps2ioUgxvCH8ItRcQwxiLGBDM2j3Yyh7GM0JQCcIKjfftMbfzF+eTo57yo72VTMRxUsi9w/RUdatK1rvmlxuLLZDkz50fEqaATMEQoaiwc+XxZHSlrmbbcsRG/vcnBo+ZamBaiDxFbr3T0AFzWwCSVrj9UOyXOS02VnoxuDT1ibL9POyn4UlInNuHdIc9LQZl3v5PW1R1CStDmuWta7wQA9fm1YbhSbSkyGOc85LV8nhSr9eXwUZjdYjXloJvDj26cCCwBKtw9olN+vOBA/ATGyxCD3ST54MpkBvA0qjkmGe0LZZ3RZT3SGiYK0NDUt7dI87ShWlRDnLuGHmqvYro6184yFd4tM75NsmFaAnSMB8D2rli,iv:MGI6Z8r7iSq607tG9zNZro9vcm1QyQfonvPDALwcGos=,tag:M50exMUMyB/XP/g74gNCwQ==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": [
+			{
+				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
+				"created_at": "2024-02-21T14:04:12Z",
+				"enc": "CiUA4OM7eKhKM5t1tiyTheQFNUS/u/5AvlqlPcTYNvKLlsPgIhHAEkkAXoW3JqdiQg/3FWiPJ9Gww+hh66YZI5UEaRalxl93S47xW0YbFcL3NFirOHbZXflEHW6Wtp+Wco9XoyL7uBmeQBKcOzLTivvo"
+			}
+		],
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2024-02-21T14:04:13Z",
+		"mac": "ENC[AES256_GCM,data:ZPkBK+cIm8zAW8Zji/bk5GzlAnz2H3maYzgiHw+dssSyAf2gVZPl8fy5+jHFfg0GfG2Cy2dwEgURNYg7Aa4UsVUq1Xkr/ppl0ZtjMqjCxs0DPK5K3jXjpA7vgvhgPCGBAaZOsA2camkAFPQrUtzWUkLq5D0LmdJGyg1bvCHKVPg=,iv:XvtMDUT3uHmw6moSNcrhhIpPHE8t8FLZl6dtkywAl6c=,tag:peyJkyp/sjAegeOnpyVllQ==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.3"
+	}
+}

terraform/aws/projects/opensci.tfvars

@@ -0,0 +1,28 @@
+region = "us-west-2"
+
+cluster_name = "opensci"
+
+cluster_nodes_location = "us-west-2a"
+
+user_buckets = {
+  "scratch-staging" : {
+    "delete_after" : 7
+  },
+  "scratch" : {
+    "delete_after" : 7
+  },
+}
+
+
+hub_cloud_permissions = {
+  "staging" : {
+    requestor_pays : true,
+    bucket_admin_access : ["scratch-staging"],
+    extra_iam_policy : ""
+  },
+  "prod" : {
+    requestor_pays : true,
+    bucket_admin_access : ["scratch"],
+    extra_iam_policy : ""
+  },
+}