MRG: merged master into current branch

.env.example

@@ -39,3 +39,9 @@ SUPERUSER_PASSWORD=
 DEFAULT_TENANT_NAME=Plio
 DEFAULT_TENANT_SHORTCODE=plio
 DEFAULT_TENANT_DOMAIN=0.0.0.0
+
+# Auth0 settings
+AUTH0_TOKEN_URL=
+AUTH0_CLIENT_ID=
+AUTH0_CLIENT_SECRET=
+AUTH0_AUDIENCE=

.github/workflows/deploy_to_ecs_prod.yml

@@ -1,8 +1,8 @@
 # This workflow will build and push a new container image to Amazon ECR
 
 on:
-  release:
-    types: [published]
+  push:
+    branches: ["release"]
 
 name: Deploy to ECS - production
 

docs/DEPLOYMENT.md

@@ -158,7 +158,7 @@ Follow the steps below to set up the staging environment on AWS.
       7. Enter `6379` as the `Port`.
       8. No need to change anything for `Parameter Group`.
       9. Choose your `Node Type`. Plio uses `cache.t2.micro`. More details [here](https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/nodes-select-size.html#CacheNodes.SelectSize).
-      10. For `Number of replicas`, enter `2`.
+      10. For `Number of replicas`, enter `0`. Note: Each replica node will carry it's own cost. More on pricing [here](https://aws.amazon.com/elasticache/pricing/) and [here](https://www.reddit.com/r/aws/comments/cojaq6/questions_about_elasticache_pricing/)
       11. For high availability, select the `Multi AZ` checkbox. Take a note that this may almost double your monthly expense. More details [here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZ.html).
       12. Select `Advanced Redis settings` and fill in the details as mentioned below.
           1. For `Subnet group`, create a new group.
@@ -246,3 +246,4 @@ Setting up a production environment on AWS is almost the same as staging. Additi
    14. Save the scaling policy.
    15. Create or update the service name.
    16. Use [k6.io](https://k6.io/) or other load testing tool to check if auto-scaling is working fine or not. You can lower down the target threshold for testing purposes.
+5.  If you're setting up [Plio Analytics](https://github.com/avantifellows/plio-analytics), also make sure to configure the required [environment variables](./ENV.md#auth0-for-plio-analytics).

docs/ENV.md

@@ -99,3 +99,19 @@ Shortcode for the default tenant (e.g. plio)
 
 #### `DEFAULT_TENANT_DOMAIN`
 The domain for the default tenant (e.g. 0.0.0.0 locally, plio.in on production)
+
+
+### Auth0 for Plio Analytics
+While setting up Plio analytics, you need to make sure the following variables are also updated. These are responsible to fetch an access token from Auth0 Identity Provider.
+
+#### `AUTH0_TOKEN_URL`
+The url to request access token from Auth0. Generally looks like `https://<AUTH0-SUBDOMAIN>.auth0.com/oauth/token`
+
+#### `AUTH0_CLIENT_ID`
+The client id for your Auth0 app. Retrieve from Auth0 Application settings page
+
+#### `AUTH0_CLIENT_SECRET`
+The client secret for your Auth0 app. Retrieve from Auth0 Application settings page
+
+#### `AUTH0_AUDIENCE`
+Unique Identifier for your Auth0 API. Retrieve from Auth0 API settings.

organizations/middleware.py

@@ -1,6 +1,7 @@
 from django.db import connection
 from django_tenants.middleware import TenantMainMiddleware
-from django_tenants.utils import get_public_schema_name, get_tenant_model
+from django_tenants.utils import get_tenant_model
+from plio.settings import DEFAULT_TENANT_SHORTCODE
 
 
 class OrganizationTenantMiddleware(TenantMainMiddleware):
@@ -9,35 +10,45 @@ class OrganizationTenantMiddleware(TenantMainMiddleware):
     """
 
     @staticmethod
-    def get_organization(request):
+    def get_organization_shortcode(request):
         """
         Returns the value of the `ORGANIZATION` HTTP header
         """
-        org = request.META.get("HTTP_ORGANIZATION", get_public_schema_name())
+        org = request.META.get("HTTP_ORGANIZATION", DEFAULT_TENANT_SHORTCODE)
         if not org:
-            return get_public_schema_name()
+            return DEFAULT_TENANT_SHORTCODE
 
         return org
 
-    def get_tenant(self, tenant_model, request):
+    def get_tenant(self, request):
         """
         Determines tenant by the value of the `ORGANIZATION` HTTP header.
         """
-        organization_shortcode = self.get_organization(request)
+        # retrieve tenant model configured in settings.py
+        tenant_model = get_tenant_model()
+
+        organization_shortcode = self.get_organization_shortcode(request)
         return tenant_model.objects.filter(shortcode=organization_shortcode).first()
 
+    def get_schema(self, request):
+        """
+        Determines the tenant schema name from the request
+        """
+        tenant = self.get_tenant(request)
+        if tenant:
+            return tenant.schema_name
+        return None
+
     def process_request(self, request):
         """
-        Switches connection to tenant schema if valid tenant. Otherwise keeps the connection with public schema.
+        Switches connection to tenant schema if valid tenant.
+        Otherwise keeps the connection with public schema.
         """
         # Connection needs first to be at the public schema, as this is where the tenant metadata is stored.
         connection.set_schema_to_public()
 
-        # retrieve tenant model configured in settings.py
-        tenant_model = get_tenant_model()
-
         # get the right tenant object based on request
-        tenant = self.get_tenant(tenant_model, request)
+        tenant = self.get_tenant(request)
         if tenant:
             # set connection to tenant's schema
             connection.set_tenant(tenant)

plio/queries.py

@@ -0,0 +1,59 @@
+def get_plio_details_query(plio_uuid: str, schema: str):
+    """Returns the details for the given plio"""
+    return f"""
+        SELECT
+            item.id AS item_id,
+            item.type AS item_type,
+            item.time AS item_time,
+            question.type AS question_type,
+            question.text AS question_text,
+            question.options AS question_options,
+            question.correct_answer AS question_correct_answer
+        FROM {schema}.plio AS plio
+        INNER JOIN {schema}.item AS item ON item.plio_id = plio.id
+        INNER JOIN {schema}.question AS question ON question.item_id = item.id
+        WHERE plio.uuid  = '{plio_uuid}'"""
+
+
+def get_sessions_dump_query(plio_uuid: str, schema: str):
+    """Returns the dump of all the sessions for the given plio"""
+    return f"""
+        SELECT
+            session.id as session_id,
+            session.retention,
+            session.watch_time,
+            MD5(session.user_id::varchar(255)) as user_id
+        FROM {schema}.session AS session
+        INNER JOIN {schema}.plio AS plio ON plio.id = session.plio_id
+        WHERE plio.uuid  = '{plio_uuid}'"""
+
+
+def get_responses_dump_query(plio_uuid: str, schema: str):
+    """Returns the dump of all the session responses for the given plio"""
+    return f"""
+        SELECT
+            session.id as session_id,
+            MD5(session.user_id::varchar(255)) as user_id,
+            sessionAnswer.id AS session_answer_id,
+            sessionAnswer.answer,
+            sessionAnswer.item_id
+        FROM {schema}.session AS session
+        INNER JOIN {schema}.session_answer sessionAnswer ON session.id = sessionAnswer.session_id
+        INNER JOIN {schema}.plio AS plio ON plio.id = session.plio_id
+        WHERE plio.uuid  = '{plio_uuid}'"""
+
+
+def get_events_query(plio_uuid: str, schema: str):
+    """Returns the dump of all events across all sessions for the given plio"""
+    return f"""
+        SELECT
+            session.id as session_id,
+            MD5(session.user_id::varchar(255)) as user_id,
+            event.type AS event_type,
+            event.player_time AS event_player_time,
+            event.details AS event_details,
+            event.created_at AS event_global_time
+        FROM {schema}.session AS session
+        INNER JOIN {schema}.event AS event ON session.id = event.session_id
+        INNER JOIN {schema}.plio AS plio ON plio.id = session.plio_id
+        WHERE plio.uuid  = '{plio_uuid}'"""

plio/settings.py

@@ -258,6 +258,8 @@
 AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "")
 AWS_REGION = os.environ.get("AWS_REGION", "")
 
+DEFAULT_TENANT_SHORTCODE = os.environ.get("DEFAULT_TENANT_SHORTCODE", "")
+
 API_APPLICATION_NAME = "plio"
 
 OAUTH2_PROVIDER = {
@@ -288,3 +290,8 @@
         "CONFIG": {"hosts": [(REDIS_HOSTNAME, REDIS_PORT)]},
     }
 }
+
+AUTH0_TOKEN_URL = os.environ.get("AUTH0_TOKEN_URL")
+AUTH0_CLIENT_ID = os.environ.get("AUTH0_CLIENT_ID")
+AUTH0_CLIENT_SECRET = os.environ.get("AUTH0_CLIENT_SECRET")
+AUTH0_AUDIENCE = os.environ.get("AUTH0_AUDIENCE")

plio/urls.py

@@ -26,6 +26,7 @@
     request_otp,
     verify_otp,
     get_by_access_token,
+    retrieve_analytics_app_access_token,
 )
 from organizations.views import OrganizationViewSet
 from experiments.views import ExperimentViewSet, ExperimentPlioViewSet
@@ -76,6 +77,7 @@
     path("api/v1/users/token/", get_by_access_token),
     path("api/v1/", include(api_router.urls)),
     path("api-auth/", include("rest_framework.urls", namespace="rest_framework")),
+    path("auth/cubejs-token/", retrieve_analytics_app_access_token),
     url(r"^auth/", include("rest_framework_social_oauth2.urls")),
     url(
         r"^api/v1/docs/$",

plio/views.py

@@ -1,19 +1,30 @@
+import os
+import shutil
 from rest_framework import viewsets, status, filters
 from rest_framework.response import Response
 from rest_framework.decorators import action
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.pagination import PageNumberPagination
-from django_tenants.utils import get_public_schema_name
+from django.db import connection
 from django.db.models import Q
-from plio.models import Video, Plio, Item, Question
+from django.http import FileResponse
+import pandas as pd
 from organizations.middleware import OrganizationTenantMiddleware
 from users.models import OrganizationUser
+from plio.models import Video, Plio, Item, Question
 from plio.serializers import (
     VideoSerializer,
     PlioSerializer,
     ItemSerializer,
     QuestionSerializer,
 )
+from plio.settings import DEFAULT_TENANT_SHORTCODE
+from plio.queries import (
+    get_plio_details_query,
+    get_sessions_dump_query,
+    get_responses_dump_query,
+    get_events_query,
+)
 
 
 class StandardResultsSetPagination(PageNumberPagination):
@@ -86,12 +97,12 @@ class PlioViewSet(viewsets.ModelViewSet):
     filter_backends = (filters.SearchFilter,)
 
     def get_queryset(self):
-        organization_shortcode = OrganizationTenantMiddleware.get_organization(
-            self.request
+        organization_shortcode = (
+            OrganizationTenantMiddleware.get_organization_shortcode(self.request)
         )
 
         # personal workspace
-        if organization_shortcode == get_public_schema_name():
+        if organization_shortcode == DEFAULT_TENANT_SHORTCODE:
             return Plio.objects.filter(created_by=self.request.user)
 
         # organizational workspace
@@ -145,7 +156,7 @@ def play(self, request, uuid):
         plio = queryset.first()
         if not plio:
             return Response(
-                {"detail": "Plio not found."}, status=status.HTTP_404_NOT_FOUND
+                {"detail": "Plio not found"}, status=status.HTTP_404_NOT_FOUND
             )
 
         serializer = self.get_serializer(plio)
@@ -162,6 +173,64 @@ def duplicate(self, request, uuid):
         plio.save()
         return Response(self.get_serializer(plio).data)
 
+    @action(methods=["get"], detail=True, permission_classes=[IsAuthenticated])
+    def download_data(self, request, uuid):
+        # return 404 if user cannot access the object
+        # else fetch the object
+        plio = self.get_object()
+
+        # handle draft plios
+        if plio.status == "draft":
+            return Response(
+                {"detail": "Data dumps are not available for draft plios"},
+                status=status.HTTP_404_NOT_FOUND,
+            )
+
+        # define the directory which will hold the data dump
+        data_dump_dir = f"/tmp/plio-{uuid}/user-{request.user.id}"
+
+        # delete the directory if it exists and create a new one
+        if os.path.exists(data_dump_dir):
+            shutil.rmtree(data_dump_dir)
+        os.makedirs(data_dump_dir)
+
+        # schema name to query in
+        schema_name = OrganizationTenantMiddleware().get_schema(self.request)
+
+        def save_query_results(cursor, query_method, filename):
+            # execute the query
+            cursor.execute(query_method(uuid, schema=schema_name))
+            # extract column names as cursor.description returns a tuple
+            columns = [col[0] for col in cursor.description]
+            # create a dataframe from the rows and the columns and save to csv
+            df = pd.DataFrame(cursor.fetchall(), columns=columns)
+            df.to_csv(os.path.join(data_dump_dir, filename), index=False)
+
+        # create the individual dump files
+        with connection.cursor() as cursor:
+            save_query_results(cursor, get_sessions_dump_query, "sessions.csv")
+            save_query_results(cursor, get_responses_dump_query, "responses.csv")
+            save_query_results(
+                cursor, get_plio_details_query, "plio-interaction-details.csv"
+            )
+            save_query_results(cursor, get_events_query, "events.csv")
+
+            df = pd.DataFrame(
+                [[plio.uuid, plio.name, plio.video.url]],
+                columns=["id", "name", "video"],
+            )
+            df.to_csv(os.path.join(data_dump_dir, "plio-meta-details.csv"), index=False)
+
+        # create the zip
+        shutil.make_archive(data_dump_dir, "zip", data_dump_dir)
+
+        # read the zip
+        zip_file = open(f"{data_dump_dir}.zip", "rb")
+
+        # create the response
+        response = FileResponse(zip_file, as_attachment=True)
+        return response
+
 
 class ItemViewSet(viewsets.ModelViewSet):
     """

users/views.py

@@ -5,6 +5,10 @@
     API_APPLICATION_NAME,
     OAUTH2_PROVIDER,
     OTP_EXPIRE_SECONDS,
+    AUTH0_TOKEN_URL,
+    AUTH0_CLIENT_ID,
+    AUTH0_CLIENT_SECRET,
+    AUTH0_AUDIENCE,
 )
 
 from rest_framework import viewsets, status
@@ -24,6 +28,8 @@
 from django.dispatch import receiver
 from django.db.models.signals import post_save, pre_save, post_delete
 
+import requests
+
 
 class UserViewSet(viewsets.ModelViewSet):
     """
@@ -202,3 +208,16 @@ def update_organization_user(sender, instance: OrganizationUser, **kwargs):
     async_to_sync(channel_layer.group_send)(
         user_group_name, {"type": "send_user", "data": user_data}
     )
+
+
+@api_view(["POST"])
+def retrieve_analytics_app_access_token(request):
+    """Makes a client_credentials request to Auth0 app to get an access token."""
+    payload = {
+        "grant_type": "client_credentials",
+        "client_id": AUTH0_CLIENT_ID,
+        "client_secret": AUTH0_CLIENT_SECRET,
+        "audience": AUTH0_AUDIENCE,
+    }
+    response = requests.post(AUTH0_TOKEN_URL, data=payload)
+    return Response(response.json(), status=status.HTTP_200_OK)