Initial PR for Vault local dev config (#2563)

* Initial PR for Vault local dev config

* Add vault client to keycloak

* Vault dev tweaks

* Remove some complexity

* Changed the redirect_uri to localhost

* Update src/ol_infrastructure/substructure/vault/secrets/__main__.py

Co-authored-by: Tobias Macey <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: Tobias Macey <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

src/ol_infrastructure/substructure/keycloak/Pulumi.substructure.keycloak.QA.yaml

@@ -40,6 +40,7 @@ config:
       airbyte: ["https://airbyte-qa.odl.mit.edu/*"]
       dagster: ["https://pipelines-qa.odl.mit.edu/*"]
       leek: ["https://celery-monitoring-qa.odl.mit.edu/*"]
+      vault: ["https://vault-qa.odl.mit.edu/ui/vault/auth/oidc/oidc/callback", "http://localhost:8250/oidc/callback"]
     realm_name: ol-platform-engineering
   - client_info:
       superset: ["https://bi-qa.ol.mit.edu/*", "superset_admin", "superset_alpha",

src/ol_infrastructure/substructure/vault/policies/developer/local_developer_policy.hcl

@@ -0,0 +1,15 @@
+path "secret-dev/*" {
+  capabilities = ["list", "read"]
+}
+
+path "secret-operations/*" {
+    capabilities = ["list"]
+}
+
+path "secret-operations/global/*" {
+    capabilities = ["list", "read"]
+}
+
+path "transit/*" {
+    capabilities = ["create", "read", "update", "delete", "list"]
+}

src/ol_infrastructure/substructure/vault/secrets/Pulumi.substructure.vault.secrets.operations.QA.yaml

@@ -3,5 +3,9 @@ secretsprovider: awskms://alias/infrastructure-secrets-qa
 encryptedkey: AQICAHg42pDDDGBhpaX14TdtzcK1hbiMYTHsYRH4k5GL5RFpIwFac25J5Xp5ipNYT8OqyxzYAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMjCzAkUqlUXy+WKJyAgEQgDviF7YJRKiSMZxleOOcxGVxZB2swKbcYMLgVaOHv9wZKGhy1HnbnLuth0cfHqWH6iisARO2bj1LxW0HSQ==
 config:
   aws:region: us-east-1
+  keycloak:client_id: ol-vault-client
+  keycloak:client_secret:
+    secure: v1:O6bloRgoHCRsGakb:QZ5JfiWqpzCeO5VqKZHRtcRkIkDEVO3RgUTQEWvAbnAZuphA7XaSZO3DiW/hpwad
+  keycloak:url: https://sso-qa.ol.mit.edu
   vault:address: https://vault-qa.odl.mit.edu
   vault_server:env_namespace: operations.qa

src/ol_infrastructure/substructure/vault/secrets/__main__.py

@@ -17,6 +17,8 @@
 )
 if Config("vault_server").get("env_namespace"):
     setup_vault_provider()
+keycloak_config = Config("keycloak")
+vault_config = Config("vault")
 
 # Create the secret mount used for storing global secrets
 global_vault_mount = vault.Mount(
@@ -36,6 +38,68 @@
         data_json=json.dumps(data),
     )
 
+# Configure secret-dev mount and keycloak auth
+if "QA" in stack_info.name:
+    # Create the secret mount used for storing env secrets for developers
+    dev_vault_mount = vault.Mount(
+        f"ol-dev-configuration-secrets-mount-{stack_info.env_suffix}",
+        path="secret-dev",
+        type="kv-v2",
+        options={"version": 2},
+        description="Storage of configuration secrets used by Devs",
+        opts=ResourceOptions(delete_before_replace=True),
+    )
+
+    # Read MIT Open vault secrets
+    mitopen_vault_secrets = read_yaml_secrets(
+        Path(f"mitopen/secrets.{stack_info.env_suffix}.yaml"),
+    )
+
+    vault.generic.Secret(
+        f"ol-dev-configuration-secrets-{stack_info.env_suffix}",
+        path=dev_vault_mount.path.apply("{}/mitopen/secrets".format),
+        data_json=json.dumps(mitopen_vault_secrets),
+    )
+
+    # Enable OIDC auth method and configure it with Keycloak
+    vault_oidc_keycloak_auth = vault.jwt.AuthBackend(
+        "vault-oidc-keycloak-backend",
+        path="oidc",
+        type="oidc",
+        description="OIDC auth Keycloak integration for use with dev vault client cli",
+        oidc_discovery_url=f"{keycloak_config.get("url")}/realms/ol-platform-engineering",
+        oidc_client_id=keycloak_config.get("client_id"),
+        oidc_client_secret=keycloak_config.get("client_secret"),
+        default_role="local-developer",
+        opts=ResourceOptions(delete_before_replace=True),
+    )
+
+    # Local Developer policy definition
+    local_developer_policy = vault.Policy(
+        "local-developer-policy",
+        name="local-developer",
+        policy=(Path(__file__).resolve())
+        .parent.parent.joinpath("policies/developer/local_developer_policy.hcl")
+        .read_text(),
+    )
+
+    # Configure OIDC role
+    local_dev_role = vault.jwt.AuthBackendRole(
+        "local-dev-role",
+        backend=vault_oidc_keycloak_auth.path,
+        role_name="local-dev",
+        token_policies=[
+            local_developer_policy.name,
+        ],
+        allowed_redirect_uris=[
+            "http://localhost:8250/oidc/callback",
+            f"{vault_config.get('address')}/ui/vault/auth/oidc/oidc/callback",
+        ],
+        bound_audiences=[keycloak_config.get("client_id")],
+        user_claim="sub",
+        role_type="oidc",
+    )
+
 vault.kv.SecretV2(
     f"grafana-vault-secrets-{stack_info.env_suffix}",
     name="grafana",