Issue 88 - first attempt, download data for a single logical device (#89

)

* WIP: Adding received_at field to REST API & DAO get messages function.

* REST API & DAO updates for this feature.

* Webapp changes to handle the geometry location columns, unit test fixes.

* Normalising the DAO get messages function to return timestamps in the message as a string. This means callers to the DAO and the REST API both get ISO-8601 format timestamp strings.

* WIP: First attempt at download data for a single logical device.

src/www/app/main.py

@@ -1,9 +1,13 @@
 import atexit
+import io
 import logging
+import pandas as pd
 import time
 from typing import Tuple
+import uuid
+from zoneinfo import ZoneInfo
 
-from flask import Flask, render_template, request, redirect, url_for, session, send_from_directory
+from flask import Flask, render_template, request, redirect, url_for, session, send_from_directory, send_file
 
 import folium
 import paho.mqtt.client as mqtt
@@ -599,6 +603,62 @@ def UpdateMappings():
         return f"HTTP request with RestAPI failed with error {e.response.status_code}", e.response.status_code
 
 
+@app.route('/download-data', methods=['POST'])
+def DownloadData():
+    try:
+        user_timezone = request.cookies.get('timezone')
+        logging.info(f'tz = {user_timezone}')
+
+        l_uid = int(request.form.get('l_uid'))
+        start_ts = request.form.get('start_ts')
+        end_ts = request.form.get('end_ts')
+
+        token = session.get('token')
+
+        logical_dev = get_logical_device(l_uid, token)
+
+        logging.info(request.form)
+        logging.info(f'start_ts = {start_ts}')
+        logging.info(f'end_ts = {end_ts}')
+        logging.info(f'l_uid = {l_uid}')
+
+        start = None
+        end = None
+        if start_ts is not None and len(start_ts) > 7:
+            start = datetime.fromisoformat(start_ts).replace(tzinfo=ZoneInfo(user_timezone))
+        if end_ts is not None and len(end_ts) > 7:
+            start = datetime.fromisoformat(start_ts).replace(tzinfo=ZoneInfo(user_timezone))
+
+        msgs = get_messages(token, l_uid, start, end)
+        logging.info(msgs)
+        if len(msgs) < 1:
+            return 'Success', 200
+
+        dataset = []
+        for msg in msgs:
+            item = {'l_uid': l_uid, 'ts': msg['timestamp'], 'received_at': msg['received_at']}
+            for obj in msg['timeseries']:
+                item[obj['name']] = obj['value']
+            dataset.append(item)
+
+        df = pd.DataFrame(dataset)
+        df.set_index(['l_uid', 'ts'], inplace=True)
+        df.sort_index(level=0, sort_remaining=True, inplace=True, ascending=True)
+
+        buffer = io.BytesIO()
+        df.to_csv(buffer, encoding='UTF-8')
+        buffer.seek(0)
+
+        return send_file(buffer, as_attachment=True, download_name=f'{logical_dev.name}.csv')
+
+
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 403:
+            return f"You do not have sufficient permissions to make this change", e.response.status_code
+
+        return f"HTTP request with RestAPI failed with error {e.response.status_code}", e.response.status_code
+
+
 @app.route('/end-ld-mapping', methods=['GET'])
 def EndLogicalDeviceMapping():
     uid = request.args['uid']
@@ -721,4 +781,7 @@ def exit_handler():
 
     atexit.register(exit_handler)
 
+    #app.jinja_env.auto_reload = True
+    #app.config['TEMPLATES_AUTO_RELOAD'] = True
+    #app.run(port='5000', host='0.0.0.0', debug=True)
     app.run(port='5000', host='0.0.0.0')

src/www/app/templates/base.html

@@ -8,14 +8,22 @@
         <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.25/css/dataTables.bootstrap5.min.css">
         <link rel="stylesheet" href="{{ url_for('static', filename='main.css') }}">
         <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+
         <script type="text/javascript" charset="utf8" src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
         <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.25/js/jquery.dataTables.min.js"></script>
         <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.25/js/dataTables.bootstrap5.min.js"></script>
         <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/select/1.4.0/js/dataTables.select.min.js"></script>
+
+        <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/flatpickr/dist/flatpickr.min.css">
+        <script src="https://cdn.jsdelivr.net/npm/flatpickr"></script>
+
         <script type="text/javascript">
             var wombatTable;
 
             $(document).ready(function () {
+                // Good to know the user's timezone when accepting dates.
+                document.cookie = "timezone=" + Intl.DateTimeFormat().resolvedOptions().timeZone + "; path=/";
+
                 $('.data-table-multi').DataTable({
                     select: {
                         style: 'multi+shift'

src/www/app/templates/logical_device_form.html

@@ -1,9 +1,158 @@
 {% extends "base.html" %}
 {% block content %}
 
+<style>
+	#export-data {
+		min-height: 500px; /* Adjust this value as needed */
+		width: 800px; /* Adjust this value as needed */
+	}
+
+	.bottom-right-div {
+		position: absolute;
+		bottom: 16px;
+		right: 16px;
+		width: 100%;
+		text-align: right;
+	  }
+</style>
+
+<script type="text/javascript">
+let start_ts_inst = null;
+let end_ts_inst = null;
+let dialog = null;
+
+$(document).ready(function () {
+	dialog = document.getElementById('export-data');
+	// A submit handler would avoid this getting called when the dialog is dismissed using
+	// the escape key, but it does not get the correct value for dialog.returnValue.
+	//
+	// The close handler gets called all the time but at least dialog.returnValue has a
+	// valid value.
+	dialog.addEventListener("close", function(event) {
+		console.log($("#start_ts").val());
+		console.log($("#end_ts").val());
+		console.log(dialog.returnValue);
+
+		if (start_ts_inst) {
+			start_ts_inst.destroy();
+		}
+		if (end_ts_inst) {
+			end_ts_inst.destroy();
+		}
+
+		if ('Export'.localeCompare(dialog.returnValue) === 0) {
+			console.log('Exporting data');
+			const formElement = document.getElementById('single-ld-export');
+			const formData = new FormData(formElement);
+			console.log(formData);
+			fetch(formElement.action, {
+				method: 'POST',
+				body: formData
+			})
+			.then(async response => {
+				if (!response.ok) {
+					throw new Error('Network response was not ok');
+				}
+				const blob = await response.blob();
+				const filename = response.headers.get('Content-Disposition')?.split('filename=')[1]?.replace(/['"]/g, '') || 'download.csv';
+
+				// Check if the showSaveFilePicker API is available
+				if ('showSaveFilePicker' in window) {
+				  try {
+					const handle = await window.showSaveFilePicker({
+					  suggestedName: filename,
+					  types: [{
+						description: 'CSV File',
+						accept: {'text/csv': ['.csv']},
+					  }],
+					});
+					const writable = await handle.createWritable();
+					await writable.write(blob);
+					await writable.close();
+				  } catch (err) {
+					if (err.name !== 'AbortError') {
+					  console.error('Failed to save file:', err);
+					  // Fallback to the older method
+					  saveBlobAsFile(blob, filename);
+					}
+				  }
+				} else {
+				  // Fallback for browsers that don't support showSaveFilePicker
+				  saveBlobAsFile(blob, filename);
+				}
+			})
+			.catch(error => {
+				console.error('Error:', error);
+				alert("Error occured on submission", error);
+			});
+		}
+	});
+});
+
+function exportData(l_uid, name) {
+	console.log("exportData " + l_uid + " " + name);
+	dialog.returnValue = "Cancel";
+	dialog.showModal();
+
+	start_ts_inst = $("#start_ts").flatpickr({
+		static: true,
+		appendTo: document.getElementById('start_ts'),
+		onChange: function(selectedDates, dateStr) {
+			$("#start_ts").val(dateStr);
+		}
+	});
+
+	end_ts_inst = $("#end_ts").flatpickr({
+		static: true,
+		appendTo: document.getElementById('end_ts'),
+		onChange: function(selectedDates, dateStr) {
+			$("#end_ts").val(dateStr);
+		}
+	});
+}
+
+function saveBlobAsFile(blob, filename) {
+	const url = window.URL.createObjectURL(blob);
+	const a = document.createElement('a');
+	a.style.display = 'none';
+	a.href = url;
+	a.download = filename;
+	document.body.appendChild(a);
+	a.click();
+	window.URL.revokeObjectURL(url);
+	document.body.removeChild(a);
+  }
+</script>
+
+<dialog id="export-data">
+	<header>
+		<h3 style="padding: 8px">Export data</h3>
+	</header>
+	<form id="single-ld-export" action="{{ url_for('DownloadData') }}" method="dialog">
+		<div>
+			<table>
+				<tr>
+					<td>From</td><td>To</td>
+				</tr>
+				<tr>
+				<td><input type="text" id="start_ts" name="start_ts" /></td>
+				<td><input type="text" id="end_ts" name="end_ts" /></td>
+				</tr>
+			</table>
+		</div>
+		<input type="hidden" name="l_uid" value="{{ ld_data.uid }}" />
+		<div class="bottom-right-div">
+			<input type="submit" value="Cancel" />
+			<input type="submit" value="Export" />
+		</div>
+	</form>
+</dialog>
+
+
 <div class="command-bar">
 	<div class="form-buttons">
 		<ul>
+			<li><span class="btn" onclick="exportData('{{ ld_data.uid }}', '{{ ld_data.name}}')">Export Data</span></li>
 			<li><span class="btn" onclick="handleSubmit('device-form', 'Are you sure you want to Save?')">Save</span></li>
 			<li><span class="btn" onclick="handleMapping('logical device')">Update Mapping</span></li>
 			<li><span class="btn" onclick="handleEndMapping('{{ ld_data.uid }}', 'LD')">End Mapping</span></li>

src/www/app/utils/api.py

@@ -1,5 +1,6 @@
+import logging
 import json, os
-from typing import List
+from typing import Any, List, Optional
 import requests
 from datetime import datetime, timezone
 import base64
@@ -190,6 +191,20 @@ def end_logical_mapping(uid: str, token: str):
     requests.patch(url, headers=headers)
 
 
+def get_messages(token: str, l_uid: int, start_ts: Optional[datetime] = None, end_ts: Optional[datetime] = None) -> List[Any]:
+    headers = {"Authorization": f"Bearer {token}"}
+    params = {"l_uid": l_uid, "include_received_at": True}
+    if start_ts is not None:
+        params['start'] = start_ts
+    if end_ts is not None:
+        params['end'] = start_ts
+
+    response = requests.get(f'{end_point}/broker/api/messages', headers=headers, params=params)
+    logging.info(response)
+    response.raise_for_status()
+    return response.json()
+
+
 def end_physical_mapping(uid: str, token: str):
     """
         End device mapping from a physical device (if any). If there was a mapping, the logical device also has no mapping after this call.