feat: example with cohere and llamacloud

examples/cohere-llamaparse/README.md

@@ -0,0 +1,15 @@
+# cohere-llamaparse
+
+This examples tests a document parsed by [LlamaCloud](https://cloud.llamaindex.ai/login) and Cohere's Command R+ model with document searching.
+
+## Usage
+
+```
+poetry install
+```
+
+Run the Empirical while defining the env variables `LLAMA_CLOUD_API_KEY` and `CO_API_KEY`.
+
+```
+LLAMA_CLOUD_API_KEY=foo CO_API_KEY=bar npx @empiricalrun/cli run --python-path `poetry env info -e`
+```

examples/cohere-llamaparse/cohere-async.py

@@ -0,0 +1,34 @@
+import httpx
+import os
+
+
+async def execute(inputs, parameters):
+    client = httpx.AsyncClient()
+    response = await client.get(
+      'https://api.cloud.llamaindex.ai/api/parsing/job/4763f2d1-5fe5-434e-abb7-b18b3e8d2577/result/markdown',
+      headers={
+        'accept': 'application/json',
+        'Authorization': f'Bearer {os.environ.get("LLAMA_CLOUD_API_KEY", "")}'
+      }
+    )
+    parsed_response = response.json()
+    doc_content = parsed_response['markdown']
+    response = await client.post(
+        'https://api.cohere.ai/v1/chat',
+        timeout=30.0, # default 10 is not working
+        headers={
+            'accept': 'application/json',
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {os.environ.get("CO_API_KEY", "")}'
+        },
+        json={
+            "preamble": "You are a helpful assistant for a gaming payments company that connects merchants (gaming apps) with their users. Users pay on the gaming apps, and this company processes their payments\n\nUsers need help regarding their transactions and have questions that they will ask you. You need to represent this company and their merchant, and talk to the user in a helpful and polite tone.\n\nUse the documents that you have to find answers, instead of jumping to answer the question yourself.",
+            "message": inputs["question"],
+            "documents": [{"title": "Client Operations.pdf", "text": doc_content}],
+        }
+    )
+    result = response.json()
+    await client.aclose()
+    return {
+        "value": result["text"]
+    }

examples/cohere-llamaparse/cohere.py

@@ -0,0 +1,31 @@
+import requests
+import os
+
+
+def execute(inputs, parameters):
+    response = requests.get(
+      'https://api.cloud.llamaindex.ai/api/parsing/job/4763f2d1-5fe5-434e-abb7-b18b3e8d2577/result/markdown',
+      headers={
+        'accept': 'application/json',
+        'Authorization': f'Bearer {os.environ.get("LLAMA_CLOUD_API_KEY", "")}'
+      }
+    )
+    parsed_response = response.json()
+    doc_content = parsed_response['markdown']
+    response = requests.post(
+        'https://api.cohere.ai/v1/chat',
+        headers={
+            'accept': 'application/json',
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {os.environ.get("CO_API_KEY", "")}'
+        },
+        json={
+            "preamble": "You are a helpful assistant for a gaming payments company that connects merchants (gaming apps) with their users. Users pay on the gaming apps, and this company processes their payments\n\nUsers need help regarding their transactions and have questions that they will ask you. You need to represent this company and their merchant, and talk to the user in a helpful and polite tone.\n\nUse the documents that you have to find answers, instead of jumping to answer the question yourself.",
+            "message": inputs["question"],
+            "documents": [{"title": "Client Operations.pdf", "text": doc_content}],
+        }
+    )
+    result = response.json()
+    return {
+        "value": result["text"]
+    }

examples/cohere-llamaparse/empiricalrc.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://assets.empirical.run/config/schema/latest.json",
+  "runs": [
+    {
+      "type": "py-script",
+      "path": "cohere.py"
+    },
+    {
+      "type": "py-script",
+      "path": "cohere-async.py"
+    }
+  ],
+  "dataset": {
+    "path": "https://docs.google.com/spreadsheets/d/1U8fBQ9TxtR5pUS0Bg0n1xcU_6EXkhGUOj1CS-TAAHmU/edit#gid=0"
+  },
+  "scorers": [
+    {
+      "type": "llm-critic",
+      "criteria": "{{ success criteria }}",
+      "name": "success-criteria"
+    }
+  ]
+}