Merge pull request #21 from mobidata-bw/push-feature

push support

.gitignore

@@ -4,8 +4,8 @@
 __pycache__/
 *.py[cod]
 .pip/
-.pytest_cache
-.coverage
+/.pytest_cache
+/.coverage*
 /htmlcov
 /reports
 /*.sql

Makefile

@@ -178,10 +178,10 @@ open-coverage:
 
 .PHONY: lint-fix
 lint-fix:
-	$(FLASK_RUN) ruff --exclude webapp/converters --fix ./webapp
-	$(FLASK_RUN) black --exclude webapp/converter ./webapp
+	$(FLASK_RUN) ruff --exclude webapp/converter --fix ./webapp
+	$(FLASK_RUN) black --exclude converter ./webapp
 
 .PHONY: lint-check
 lint-check:
 	$(FLASK_RUN) ruff --exclude webapp/converter ./webapp
-	$(FLASK_RUN) black --exclude webapp/converter -S --check --diff webapp
+	$(FLASK_RUN) black --exclude converter -S --check --diff webapp

dev/api_tests/public_api/park_api_v1.http

@@ -1,10 +1,10 @@
 ### List park api v1 overview data
-GET {{api_host}}/api/public/v1/park-api-v1
+GET {{api_host}}/api/public/v1
 
 
 ### List park api v1 detail data
-GET {{api_host}}/api/public/v1/park-api-v1/p-r-vrs
+GET {{api_host}}/api/public/v1/p-r-vrs
 
 
 ### List park api v1 detail data with name filter
-GET {{api_host}}/api/public/v1/park-api-v1/example-source?name=Bahnhof
+GET {{api_host}}/api/public/v1/example-source?name=Bahnhof

dev/api_tests/public_api/park_api_v2.http

@@ -1,14 +1,14 @@
 ### List park api v2 detail data
-GET {{api_host}}/api/public/v1/park-api-v2/lots/
+GET {{api_host}}/api/public/v2/lots/
 
 
 ### List park api v2 detail data with location filter
-GET {{api_host}}/api/public/v1/park-api-v2/lots/?location=12,50&radius=1000
+GET {{api_host}}/api/public/v2/lots/?location=12,50&radius=1000
 
 
 ### List park api v2 detail data with name filter
-GET {{api_host}}/api/public/v1/park-api-v2/lots/?name=Bahnhof
+GET {{api_host}}/api/public/v2/lots/?name=Bahnhof
 
 
 ### List park api v2 pool data
-GET {{api_host}}/api/public/v1/park-api-v2/pools/example-source/
+GET {{api_host}}/api/public/v2/pools/example-source/

dev/api_tests/public_api/parking-sites.http

@@ -1,10 +1,10 @@
 ### List parking sites
-GET {{api_host}}/api/public/v1/parking-sites
+GET {{api_host}}/api/public/v3/parking-sites
 
 
 ### List parking sites with name filter
-GET {{api_host}}/api/public/v1/parking-sites?name=Bahnhof
+GET {{api_host}}/api/public/v3/parking-sites?name=Bahnhof
 
 
 ### Get single parking site
-GET {{api_host}}/api/public/v1/parking-sites/1
+GET {{api_host}}/api/public/v3/parking-sites/1

docker-compose.yml

@@ -21,6 +21,8 @@ x-flask-defaults: &flask-defaults
   depends_on:
     postgresql:
       condition: service_healthy
+    mysql:
+      condition: service_healthy
     rabbitmq:
       condition: service_healthy
 

push-client/push-client.py

@@ -0,0 +1,68 @@
+"""
+Copyright 2023 binary butterfly GmbH
+Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
+"""
+
+import argparse
+import sys
+from getpass import getpass
+from pathlib import Path
+
+import requests
+
+DATA_TYPES = {
+    'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    'csv': 'text/csv',
+    'xml': 'application/xml',
+    'json': 'application/json',
+}
+
+PUSH_BASE_URL = 'http://localhost:5000/api/admin/v1/generic-parking-sites'
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog='ParkAPI Push Client',
+        description='This client helps to push static ParkAPI data',
+    )
+    parser.add_argument('source_uid')
+    parser.add_argument('file_path')
+    args = parser.parse_args()
+    source_uid: str = args.source_uid
+    file_path: Path = Path(args.file_path)
+
+    if not file_path.is_file():
+        sys.exit('Error: please add a file as second argument.')
+
+    password = getpass(f'Password for source UID {source_uid}: ')
+
+    file_ending = None
+    for ending in DATA_TYPES:
+        if file_path.name.endswith(f'.{ending}'):
+            file_ending = ending
+
+    if file_ending is None:
+        sys.exit(f'Error: invalid ending. Allowed endings are: {", ".join(DATA_TYPES.keys())}')
+
+    with file_path.open('rb') as file:
+        file_data = file.read()
+
+    endpoint = f'{PUSH_BASE_URL}/{file_ending}'
+    requests_response = requests.post(
+        url=endpoint,
+        data=file_data,
+        auth=(source_uid, password),
+        headers={'Content-Type': DATA_TYPES[file_ending]},
+    )
+
+    if requests_response.status_code == 204:
+        sys.exit('Upload successful.')
+
+    if requests_response.status_code == 401:
+        sys.exit('Access denied.')
+
+    sys.exit(f'Unknown error with HTTP status code {requests_response.status_code}.')
+
+
+if __name__ == "__main__":
+    main()

requirements-dev.txt

@@ -1,10 +1,3 @@
-pytest~=7.4.2
-pytest-cov~=4.1.0
+pytest~=7.4.3
 black~=23.10.0
-mypy~=1.6.1
-types-Flask-Migrate~=4.0.0.6
-types-PyYAML~=6.0.12.12
-types-requests~=2.31.0.10
-requests-mock~=1.11.0
-mypy-gitlab-code-quality~=1.0.0
-ruff~=0.1.0
+ruff~=0.1.3

requirements.txt

@@ -19,6 +19,7 @@ click~=8.1.7
 openpyxl~=3.1.2
 opening-hours-py~=0.6.17
 kombu~=5.3.2
+lxml~=4.9.3
 
 # required for converters
 beautifulsoup4~=4.12.2

webapp/admin_rest_api/admin_rest_api.py

@@ -3,7 +3,12 @@
 Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
 """
 
+from flask import Response, request
+
 from webapp.common.blueprint import Blueprint
+from webapp.common.logging import Logger
+from webapp.common.logging.models import LogMessageType, LogTag
+from webapp.dependencies import dependencies
 
 from .base_blueprint import AdminApiBaseBlueprint
 from .generic_parking_sites import GenericParkingSitesBlueprint
@@ -22,3 +27,27 @@ def __init__(self):
 
         for blueprint_class in self.blueprints_classes:
             self.register_blueprint(blueprint_class())
+
+        @self.before_request
+        def before_request(*args, **kwargs):
+            logger: Logger = dependencies.get_logger()
+            logger.set_tag(LogTag.INITIATOR, 'admin-api')
+
+        @self.after_request
+        def after_request(response: Response):
+            if not request.path.startswith('/api/admin/v1'):
+                return response
+
+            log_fragments = [f'{request.method.upper()} {request.full_path}: HTTP {response.status}']
+            if request.data:
+                if request.mimetype == 'application/json':
+                    log_fragments.append(f'>> {request.data.decode()}')
+                else:
+                    log_fragments.append(f'>> binary data with {len(request.data)} byte')
+            if response.data and response.data.decode().strip():
+                log_fragments.append(f'<< {response.data.decode().strip()}')
+
+            logger: Logger = dependencies.get_logger()
+            logger.info(LogMessageType.REQUEST_IN, '\n'.join(log_fragments))
+
+            return response

webapp/admin_rest_api/base_blueprint.py

@@ -6,6 +6,9 @@
 from flask import current_app, request
 
 from webapp.common.blueprint import Blueprint
+from webapp.common.logging import Logger
+from webapp.common.logging.models import LogTag
+from webapp.common.server_auth import ServerAuthHelper
 from webapp.dependencies import dependencies
 
 
@@ -26,8 +29,11 @@ def before_request():
                 ):
                     return
                 # Authenticate user via Basic Auth (raises AdminApiUnauthorizedException if unauthenticated)
-                server_auth_helper = dependencies.get_server_auth_helper()
+                server_auth_helper: ServerAuthHelper = dependencies.get_server_auth_helper()
                 server_auth_helper.authenticate_request(request)
+                logger: Logger = dependencies.get_logger()
+                logger.set_tag(LogTag.INITIATOR, 'admin-api')
+                logger.set_tag(LogTag.USER, server_auth_helper.get_current_user().username)
 
     @staticmethod
     def get_base_handler_dependencies() -> dict:

webapp/admin_rest_api/base_handler.py

@@ -5,7 +5,7 @@
 
 from webapp.common.config import ConfigHelper
 from webapp.common.events import EventHelper
-from webapp.common.logger import Logger
+from webapp.common.logging import Logger
 
 
 class AdminApiBaseHandler:

webapp/admin_rest_api/generic_parking_sites/generic_parking_sites_handler.py

@@ -3,30 +3,113 @@
 Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
 """
 
+import csv
+from io import BytesIO, StringIO
+
+from lxml import etree
+from openpyxl.reader.excel import load_workbook
+
 from webapp.admin_rest_api import AdminApiBaseHandler
+from webapp.common.logging.models import LogTag
+from webapp.common.rest.exceptions import RestApiNotImplementedException
+from webapp.models import ParkingSite, Source
+from webapp.models.parking_site import ParkingSiteType
 from webapp.repositories import ParkingSiteRepository, SourceRepository
+from webapp.repositories.exceptions import ObjectNotFoundException
+from webapp.services.import_service import ParkingSiteGenericImportService
 
 
 class GenericParkingSitesHandler(AdminApiBaseHandler):
     source_repository: SourceRepository
     parking_site_repository: ParkingSiteRepository
+    parking_site_generic_import_service: ParkingSiteGenericImportService
 
     def __init__(
         self,
         *args,
         source_repository: SourceRepository,
         parking_site_repository: ParkingSiteRepository,
+        parking_site_generic_import_service: ParkingSiteGenericImportService,
         **kwargs,
     ):
         super().__init__(*args, **kwargs)
         self.source_repository = source_repository
         self.parking_site_repository = parking_site_repository
+        self.parking_site_generic_import_service = parking_site_generic_import_service
+
+    def handle_json_data(self, source_uid: str, data: dict | list):
+        source = self._get_source(source_uid)
+        import_service = self.parking_site_generic_import_service.push_converters[source_uid]
+
+        import_results = import_service.handle_json(data)
+        static_parking_site_inputs = import_results.static_parking_site_inputs
+
+        for static_parking_site_input in static_parking_site_inputs:
+            self._save_parking_site_input(source, static_parking_site_input)
+
+    def handle_xml_data(self, source_uid: str, data: str):
+        source = self._get_source(source_uid)
+        import_service = self.parking_site_generic_import_service.push_converters[source_uid]
+
+        root_element = etree.parse(StringIO(data), parser=etree.XMLParser(resolve_entities=False))  # noqa: S320
+        import_results = import_service.handle_xml(root_element)
+        static_parking_site_inputs = import_results.static_parking_site_inputs
+
+        for static_parking_site_input in static_parking_site_inputs:
+            self._save_parking_site_input(source, static_parking_site_input)
+
+    def handle_csv_data(self, source_uid: str, data: str):
+        source = self._get_source(source_uid)
+        import_service = self.parking_site_generic_import_service.push_converters[source_uid]
+
+        rows = list(csv.reader(StringIO(data)))
+        import_results = import_service.handle_csv(rows)
+        static_parking_site_inputs = import_results.static_parking_site_inputs
+
+        for static_parking_site_input in static_parking_site_inputs:
+            self._save_parking_site_input(source, static_parking_site_input)
+
+    def handle_xlsx_data(self, source_uid: str, data: bytes):
+        source = self._get_source(source_uid)
+        import_service = self.parking_site_generic_import_service.push_converters[source_uid]
+
+        workbook = load_workbook(filename=BytesIO(data))
+        import_results = import_service.handle_xlsx(workbook)
+        static_parking_site_inputs = import_results.static_parking_site_inputs
+
+        for static_parking_site_input in static_parking_site_inputs:
+            self._save_parking_site_input(source, static_parking_site_input)
+
+    def _get_source(self, source_uid: str) -> Source:
+        try:
+            source = self.source_repository.fetch_source_by_uid(source_uid)
+        except ObjectNotFoundException:
+            source = Source()
+            source.uid = source_uid
+            self.source_repository.save_source(source)
+        self.logger.set_tag(LogTag.SOURCE, source_uid)
+
+        if source_uid not in self.parking_site_generic_import_service.push_converters:
+            raise RestApiNotImplementedException(message='Converter is missing for this source.')
+
+        return source
 
-    def handle_json_data(self, data: dict | list):
-        pass
+    def _save_parking_site_input(self, source: Source, static_parking_site_input):
+        try:
+            parking_site = self.parking_site_repository.fetch_parking_site_by_source_id_and_external_uid(
+                source_id=source.id,
+                original_uid=static_parking_site_input.uid,
+            )
+        except ObjectNotFoundException:
+            parking_site = ParkingSite()
+            parking_site.source_id = source.id
+            parking_site.original_uid = static_parking_site_input.uid
 
-    def handle_csv_data(self, data: bytes):
-        pass
+        for key, value in static_parking_site_input.to_dict().items():
+            if key in ['uid']:
+                continue
+            if key == 'type' and value:
+                value = ParkingSiteType[value.name]
+            setattr(parking_site, key, value)
 
-    def handle_xlsx_data(self, data: bytes):
-        pass
+        self.parking_site_repository.save_parking_site(parking_site)