From 50573200deb50611eef7ce142aeb270a535eb50a Mon Sep 17 00:00:00 2001 From: Klaas Schoute Date: Mon, 5 Dec 2022 02:56:29 +0100 Subject: [PATCH] Move to new package set for Amsterdam (#206) --- app/cities/netherlands/amsterdam.py | 40 ++++++++++++++++------------- requirements.txt | 1 + run.py | 31 +++++++++++----------- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/app/cities/netherlands/amsterdam.py b/app/cities/netherlands/amsterdam.py index c7b56b7..e30f74d 100644 --- a/app/cities/netherlands/amsterdam.py +++ b/app/cities/netherlands/amsterdam.py @@ -1,9 +1,8 @@ """Manage the location data of Amsterdam.""" import datetime -import json -import aiohttp import pytz +from odp_amsterdam import ODPAmsterdam from app.database import connection, cursor from app.helper import centroid @@ -13,13 +12,15 @@ CBS_CODE = "0363" -async def async_get_locations(): - """Get the data from the GeoJSON API endpoint.""" - async with aiohttp.ClientSession() as client: - async with client.get( - "https://api.data.amsterdam.nl/v1/parkeervakken/parkeervakken?eType=E6a&_format=geojson" - ) as resp: - return await resp.text() +async def async_get_locations(limit): + """Get parking data from API. + + Args: + limit (int): The number of parking lots to get. + """ + async with ODPAmsterdam() as client: + locations = await client.locations(limit=limit, parking_type="E6a") + return locations def correct_orientation(orientation_type) -> str: @@ -37,17 +38,19 @@ def correct_orientation(orientation_type) -> str: def upload(data_set): - """Upload the data from the JSON file to the database.""" - amsterdam_obj = json.loads(data_set) + """Upload the data from the JSON file to the database. + + Args: + data_set: The data_set to upload. + """ index: int try: - for index, item in enumerate(amsterdam_obj["features"], 1): + for index, item in enumerate(data_set, 1): # Get the coordinates of the parking lot with centroid - latitude, longitude = centroid(item["geometry"]["coordinates"]) + latitude, longitude = centroid(item.coordinates) # Define unique id - location_id = f"{GEOCODE}-{CBS_CODE}-{item['id'].split('.')[1]}" + location_id = f"{GEOCODE}-{CBS_CODE}-{item.spot_id}" - item = item["properties"] # Make the sql query sql = """INSERT INTO `parking_cities` (id, country_id, province_id, municipality, street, orientation, number, longitude, latitude, visibility, created_at, updated_at) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY @@ -66,15 +69,16 @@ def upload(data_set): int(157), int(8), str(MUNICIPALITY), - str(item["straatnaam"]), - correct_orientation(item["type"]), - int(item["aantal"]), + str(item.street), + correct_orientation(item.orientation), + int(item.number), float(longitude), float(latitude), bool(True), (datetime.datetime.now(tz=pytz.timezone("Europe/Amsterdam"))), (datetime.datetime.now(tz=pytz.timezone("Europe/Amsterdam"))), ) + # print(val) cursor.execute(sql, val) connection.commit() except Exception as error: diff --git a/requirements.txt b/requirements.txt index 64a8376..7e9564b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,6 +32,7 @@ multidict==6.0.2 mypy-extensions==0.4.3 namur==0.1.1 nodeenv==1.7.0 +odp-amsterdam==5.0.0 pathspec==0.10.2 platformdirs==2.5.4 pre-commit==2.20.0 diff --git a/run.py b/run.py index d6d7d09..34cc24b 100644 --- a/run.py +++ b/run.py @@ -16,8 +16,7 @@ groningen, zoetermeer, ) - -# import app.database as database +from app.database import truncate # pylint: disable=unused-import # noqa: F401 if __name__ == "__main__": load_dotenv() @@ -29,88 +28,88 @@ print("--- Start program ---") if city == "amsterdam": # Amsterdam - NL - data_set = asyncio.run(amsterdam.async_get_locations()) + data_set = asyncio.run(amsterdam.async_get_locations(limit=2000)) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Amsterdam") + # truncate("Amsterdam") amsterdam.upload(data_set) elif city == "arnhem": # Arnhem - NL arnhem.download() - # database.truncate("Arnhem") + # truncate("Arnhem") arnhem.upload() elif city == "amersfoort": # Amersfoort -NL data_set = asyncio.run(amersfoort.async_get_locations()) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Amersfoort") + # truncate("Amersfoort") amersfoort.upload(data_set) elif city == "brussel": # Brussels - BE data_set = asyncio.run(brussel.async_get_locations(limit=1000)) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Brussel") + # truncate("Brussel") brussel.upload(data_set) elif city == "denhaag": # Den Haag - NL data_set = asyncio.run(den_haag.async_get_locations(limit=300)) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Den Haag") + # truncate("Den Haag") den_haag.upload(data_set) elif city == "dusseldorf": # Dusseldorf - DE data_set = asyncio.run(dusseldorf.async_get_locations(limit=350)) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Dusseldorf") + # truncate("Dusseldorf") dusseldorf.upload(data_set) elif city == "dresden": # Dresden - DE data_set = asyncio.run(dresden.async_get_locations(limit=500)) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Dresden") + # truncate("Dresden") dresden.upload(data_set) elif city == "eindhoven": # Eindhoven - NL data_set = asyncio.run(eindhoven.async_get_locations(limit=200)) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Eindhoven") + # truncate("Eindhoven") eindhoven.upload(data_set) elif city == "hamburg": # Hamburg - DE data_set = asyncio.run(hamburg.async_get_locations()) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Hamburg") + # truncate("Hamburg") hamburg.upload(data_set) elif city == "liege": # Liege - BE data_set = asyncio.run(liege.async_get_locations(limit=1000)) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Liege") + # truncate("Liege") liege.upload(data_set) elif city == "namur": # Namur - BE data_set = asyncio.run(namur.async_get_locations(limit=1000)) if data_set: print(f"Data retrieved from: {city}") - # database.truncate("Namur") + # truncate("Namur") namur.upload(data_set) elif city == "groningen": # Groningen groningen.download() - # database.truncate("Groningen") + # truncate("Groningen") groningen.upload() elif city == "zoetermeer": # Zoetermeer zoetermeer.download() - # database.truncate("Zoetermeer") + # truncate("Zoetermeer") zoetermeer.upload() else: print(f"{city} is currently not supported")