feature: Reutlingen bike converter, enable CI tests

.github/workflows/lint.yml → .github/workflows/lint-test.yml

@@ -13,17 +13,15 @@ on:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version:
-          - '3'
+
     steps:
     - name: checkout
       uses: actions/checkout@v4
-    - name: setup Python v${{ matrix.python-version }}
+
+    - name: setup Python v3.10
       uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: '3.10'
         cache: 'pip'
 
     - name: pip install
@@ -41,3 +39,28 @@ jobs:
       # uses: uses: psf/black@stable
       run: |
         black -S --check --diff ./src ./tests
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+
+    - name: set timezone
+      uses: szenius/[email protected]
+      with:
+        timezoneLinux: "Europe/Berlin"
+
+    - name: checkout
+      uses: actions/checkout@v4
+
+    - name: setup Python v3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+        cache: 'pip'
+
+    - name: pip install
+      run: pip install -r requirements.txt -r requirements-dev.txt
+
+    - name: run pytest
+      run: python -m pytest tests

README.md

@@ -20,7 +20,8 @@ We support following data sources:
 | Stadt Karlsruhe                                                                   | car     | push (csv)  | `neckarsulm`         | no       |
 | Parkraumgesellschaft Baden-Württemberg                                            | car     | pull        | `pbw`                | yes      |
 | Stadt Pforzheim                                                                   | car     | push (csv)  | `pforzheim`          | no       |
-| Stadt Reutlingen                                                                  | car     | push (csv)  | `reutlingen`         | no       |
+| Stadt Reutlingen: PKW-Parkplätze                                                  | car     | push (csv)  | `reutlingen`         | no       |
+| Stadt Reutlingen: Fahrrad-Abstellanlagen                                          | bike    | push (csv)  | `reutlingen_bike`    | no       |
 | Stadt Stuttgart                                                                   | car     | push (json) | `stuttgart`          | yes      |
 | Stadt Ulm                                                                         | car     | pull        | `ulm`                | yes      |
 | Verband Region Stuttgart: Park and Ride                                           | car     | pull        | `vrs_p_r`            | yes      |

src/parkapi_sources/converters/__init__.py

@@ -14,6 +14,7 @@
 from .pbw import PbwPullConverter
 from .pforzheim import PforzheimPushConverter
 from .reutlingen import ReutlingenPushConverter
+from .reutlingen_bike import ReutlingenBikePushConverter
 from .stuttgart import StuttgartPushConverter
 from .ulm import UlmPullConverter
 from .vrs_p_r import VrsParkAndRidePushConverter

src/parkapi_sources/converters/reutlingen/converter.py

@@ -4,7 +4,6 @@
 """
 
 import csv
-from datetime import datetime, timezone
 from io import StringIO
 
 from validataclass.exceptions import ValidationError
@@ -22,7 +21,7 @@ class ReutlingenPushConverter(CsvConverter):
 
     source_info = SourceInfo(
         uid='reutlingen',
-        name='Stadt Reutlingen',
+        name='Stadt Reutlingen: PKW-Parkplätze',
         public_url='https://www.reutlingen.de',
         has_realtime_data=False,
     )
@@ -48,26 +47,17 @@ def handle_csv(self, data: list[list]) -> tuple[list[StaticParkingSiteInput], li
                 input_dict[field] = row[mapping[field]]
 
             try:
-                input_data: ReutlingenRowInput = self.reutlingen_row_validator.validate(input_dict)
+                reutlingen_row_input: ReutlingenRowInput = self.reutlingen_row_validator.validate(input_dict)
             except ValidationError as e:
                 static_parking_site_errors.append(
                     ImportParkingSiteException(
-                        uid=input_dict.get('uid'),
+                        source_uid=self.source_info.uid,
+                        parking_site_uid=input_dict.get('uid'),
                         message=f'validation error for {input_dict}: {e.to_dict()}',
                     ),
                 )
                 continue
 
-            parking_site_input = StaticParkingSiteInput(
-                uid=str(input_data.uid),
-                name=input_data.name,
-                address=f'{input_data.name}, Reutlingen',
-                lat=input_data.coordinates[1],
-                lon=input_data.coordinates[0],
-                type=input_data.type.to_parking_site_type_input(),
-                capacity=input_data.capacity,
-                static_data_updated_at=datetime.now(tz=timezone.utc),
-            )
-            static_parking_site_inputs.append(parking_site_input)
+            static_parking_site_inputs.append(reutlingen_row_input.to_parking_site_input())
 
         return static_parking_site_inputs, static_parking_site_errors

src/parkapi_sources/converters/reutlingen/validation.py

@@ -3,16 +3,15 @@
 Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
 """
 
-import re
+from datetime import datetime, timezone
 from enum import Enum
-from typing import Any
 
 from validataclass.dataclasses import validataclass
-from validataclass.exceptions import ValidationError
-from validataclass.validators import DecimalValidator, EnumValidator, IntegerValidator, ListValidator, StringValidator
+from validataclass.validators import DecimalValidator, EnumValidator, IntegerValidator, StringValidator
 
+from parkapi_sources.models import StaticParkingSiteInput
 from parkapi_sources.models.enums import ParkingSiteType
-from parkapi_sources.validators import ExcelNoneable
+from parkapi_sources.validators import PointCoordinateTupleValidator
 
 
 class ReutlingenParkingSiteType(Enum):
@@ -29,25 +28,22 @@ def to_parking_site_type_input(self) -> ParkingSiteType:
         }.get(self, ParkingSiteType.OTHER)
 
 
-class PointCoordinateTupleValidator(ListValidator):
-    PATTERN = re.compile(r'POINT \(([-+]?\d+\.\d+) ([-+]?\d+\.\d+)\)')
-
-    def validate(self, input_data: Any, **kwargs) -> list:
-        self._ensure_type(input_data, str)
-        input_match = re.match(self.PATTERN, input_data)
-
-        if input_match is None:
-            raise ValidationError(code='invalid_tuple_input', reason='invalid point coordinate tuple input')
-
-        input_data = [input_match.group(1), input_match.group(2)]
-
-        return super().validate(input_data, **kwargs)
-
-
 @validataclass
 class ReutlingenRowInput:
     uid: int = IntegerValidator(allow_strings=True)
     type: ReutlingenParkingSiteType = EnumValidator(ReutlingenParkingSiteType)
     coordinates: list = PointCoordinateTupleValidator(DecimalValidator())
-    capacity: str = ExcelNoneable(IntegerValidator(allow_strings=True))
+    capacity: int = IntegerValidator(allow_strings=True)
     name: str = StringValidator(max_length=255)
+
+    def to_parking_site_input(self) -> StaticParkingSiteInput:
+        return StaticParkingSiteInput(
+            uid=str(self.uid),
+            name=self.name,
+            address=f'{self.name}, Reutlingen',
+            lat=self.coordinates[1],
+            lon=self.coordinates[0],
+            type=self.type.to_parking_site_type_input(),
+            capacity=self.capacity,
+            static_data_updated_at=datetime.now(tz=timezone.utc),
+        )

src/parkapi_sources/converters/reutlingen_bike/__init__.py

@@ -0,0 +1,6 @@
+"""
+Copyright 2024 binary butterfly GmbH
+Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
+"""
+
+from .converter import ReutlingenBikePushConverter

src/parkapi_sources/converters/reutlingen_bike/converter.py

@@ -0,0 +1,69 @@
+"""
+Copyright 2024 binary butterfly GmbH
+Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
+"""
+
+import csv
+from io import StringIO
+
+import pyproj
+from validataclass.exceptions import ValidationError
+from validataclass.validators import DataclassValidator
+
+from parkapi_sources.converters.base_converter.push import CsvConverter
+from parkapi_sources.converters.reutlingen_bike.validation import ReutlingenBikeRowInput
+from parkapi_sources.exceptions import ImportParkingSiteException
+from parkapi_sources.models import RealtimeParkingSiteInput, SourceInfo, StaticParkingSiteInput
+
+
+class ReutlingenBikePushConverter(CsvConverter):
+    proj: pyproj.Proj = pyproj.Proj(proj='utm', zone=32, ellps='WGS84', preserve_units=True)
+    reutlingen_bike_row_validator = DataclassValidator(ReutlingenBikeRowInput)
+
+    source_info = SourceInfo(
+        uid='reutlingen',
+        name='Stadt Reutlingen: Fahrrad-Abstellanlagen',
+        public_url='https://www.reutlingen.de',
+        has_realtime_data=False,
+    )
+
+    header_mapping: dict[str, str] = {
+        '\ufeffSTANDORT': 'name',
+        'ANZAHL': 'capacity',
+        'ANLAGE': 'additional_name',
+        'GEOM': 'coordinates',
+    }
+
+    def handle_csv_string(
+        self,
+        data: StringIO,
+    ) -> tuple[list[StaticParkingSiteInput | RealtimeParkingSiteInput], list[ImportParkingSiteException]]:
+        return self.handle_csv(list(csv.reader(data, dialect='unix', delimiter=',')))
+
+    def handle_csv(self, data: list[list]) -> tuple[list[StaticParkingSiteInput], list[ImportParkingSiteException]]:
+        static_parking_site_inputs: list[StaticParkingSiteInput] = []
+        static_parking_site_errors: list[ImportParkingSiteException] = []
+
+        mapping: dict[str, int] = self.get_mapping_by_header(self.header_mapping, data[0])
+
+        # We start at row 2, as the first one is our header
+        for row in data[1:]:
+            input_dict: dict[str, str] = {}
+            for field in self.header_mapping.values():
+                input_dict[field] = row[mapping[field]]
+
+            try:
+                reutlingen_bike_row_input: ReutlingenBikeRowInput = self.reutlingen_bike_row_validator.validate(input_dict)
+            except ValidationError as e:
+                static_parking_site_errors.append(
+                    ImportParkingSiteException(
+                        source_uid=self.source_info.uid,
+                        parking_site_uid=input_dict.get('name'),
+                        message=f'validation error for {input_dict}: {e.to_dict()}',
+                    ),
+                )
+                continue
+
+            static_parking_site_inputs.append(reutlingen_bike_row_input.to_parking_site_input(self.proj))
+
+        return static_parking_site_inputs, static_parking_site_errors

src/parkapi_sources/converters/reutlingen_bike/validation.py

@@ -0,0 +1,43 @@
+"""
+Copyright 2024 binary butterfly GmbH
+Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
+"""
+
+from datetime import datetime, timezone
+
+import pyproj
+from validataclass.dataclasses import validataclass
+from validataclass.validators import DecimalValidator, IntegerValidator, StringValidator
+
+from parkapi_sources.models import StaticParkingSiteInput
+from parkapi_sources.models.enums import PurposeType
+from parkapi_sources.validators import PointCoordinateTupleValidator
+
+
+@validataclass
+class ReutlingenBikeRowInput:
+    coordinates: list = PointCoordinateTupleValidator(DecimalValidator())
+    capacity: int = IntegerValidator(allow_strings=True)
+    name: str = StringValidator(max_length=255)
+    additional_name: str = StringValidator(max_length=255)
+
+    def to_parking_site_input(self, proj: pyproj.Proj) -> StaticParkingSiteInput:
+        coordinates = proj(float(self.coordinates[0]), float(self.coordinates[1]), inverse=True)
+        lat = coordinates[1]
+        lon = coordinates[0]
+
+        if self.name and self.additional_name:
+            name = f'{self.name}, {self.additional_name}'
+        elif self.additional_name:
+            name = self.additional_name
+        else:
+            name = self.name
+        return StaticParkingSiteInput(
+            uid=f'{name}: {lat}-{lon}',
+            lat=lat,
+            lon=lon,
+            name=name,
+            static_data_updated_at=datetime.now(tz=timezone.utc),
+            capacity=self.capacity,
+            purpose=PurposeType.BIKE,
+        )

src/parkapi_sources/validators/__init__.py

@@ -8,6 +8,7 @@
 from .datetime_validator import Rfc1123DateTimeValidator, SpacedDateTimeValidator
 from .decimal_validators import GermanDecimalValidator
 from .integer_validators import GermanDurationIntegerValidator
+from .list_validator import PointCoordinateTupleValidator
 from .noneable import ExcelNoneable
 from .string_validators import NumberCastingStringValidator, ReplacingStringValidator
 from .time_validators import ExcelTimeValidator

src/parkapi_sources/validators/list_validator.py

@@ -0,0 +1,25 @@
+"""
+Copyright 2024 binary butterfly GmbH
+Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
+"""
+
+import re
+from typing import Any
+
+from validataclass.exceptions import ValidationError
+from validataclass.validators import ListValidator
+
+
+class PointCoordinateTupleValidator(ListValidator):
+    PATTERN = re.compile(r'POINT \(([-+]?\d+\.\d+) ([-+]?\d+\.\d+)\)')
+
+    def validate(self, input_data: Any, **kwargs) -> list:
+        self._ensure_type(input_data, str)
+        input_match = re.match(self.PATTERN, input_data)
+
+        if input_match is None:
+            raise ValidationError(code='invalid_tuple_input', reason='invalid point coordinate tuple input')
+
+        input_data = [input_match.group(1), input_match.group(2)]
+
+        return super().validate(input_data, **kwargs)