From 291a36c115ebb25456c1a99e9e12f8601c5df9ed Mon Sep 17 00:00:00 2001 From: Klaas Schoute Date: Sat, 9 Oct 2021 05:15:32 +0200 Subject: [PATCH] Update code (#2) * Update packages * Update code per city * Fix typo in filename --- .github/{FUNDING.yaml => FUNDING.yml} | 0 __init__.py | 8 +++++++- cities/amsterdam.py | 15 ++++----------- cities/den_haag.py | 17 ++++------------- cities/zoetermeer.py | 18 ++++++------------ database/__init__.py | 17 ++++++++++++++++- requirements.txt | 12 ++++++++++++ 7 files changed, 49 insertions(+), 38 deletions(-) rename .github/{FUNDING.yaml => FUNDING.yml} (100%) diff --git a/.github/FUNDING.yaml b/.github/FUNDING.yml similarity index 100% rename from .github/FUNDING.yaml rename to .github/FUNDING.yml diff --git a/__init__.py b/__init__.py index ca0fc27..15d2d2e 100644 --- a/__init__.py +++ b/__init__.py @@ -3,13 +3,19 @@ import cities.arnhem as arnhem import cities.den_haag as den_haag import cities.amsterdam as amsterdam +import database as database +import asyncio if __name__ == '__main__': - print("Start program") + print("--- Start program ---") + """ Zoetermeer """ # zoetermeer.download() # zoetermeer.upload() # zoetermeer.truncate() # arnhem.download() + """ Amsterdam """ # amsterdam.upload() + + """ Den Haag """ # den_haag.upload() \ No newline at end of file diff --git a/cities/amsterdam.py b/cities/amsterdam.py index b7a5924..67c0b3f 100644 --- a/cities/amsterdam.py +++ b/cities/amsterdam.py @@ -5,6 +5,8 @@ city = "Amsterdam" def upload(): + """Upload the data from the JSON file to the database.""" + amsterdam_file = "data/parking-amsterdam.json" amsterdam_data = open(amsterdam_file).read() amsterdam_obj = json.loads(amsterdam_data) @@ -19,10 +21,11 @@ def upload(): latitude = latitude.replace(']', '') id = uuid.uuid4().hex[:8] + item = item["properties"] sql = """INSERT INTO `parking_cities` (`id`, `city`, `street`, `orientation`, `number`, `longitude`, `latitude`, `visibility`, `created_at`, `updated_at`) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""" - val = (id, str(city), str(item["properties"]["straatnaam"]), str(item["properties"]["type"]), int(item["properties"]["aantal"]), float(longitude), float(latitude), bool(True), (datetime.datetime.now()), (datetime.datetime.now())) + val = (id, str(city), str(item["straatnaam"]), str(item["type"]), int(item["aantal"]), float(longitude), float(latitude), bool(True), (datetime.datetime.now()), (datetime.datetime.now())) cursor.execute(sql, val) connection.commit() except Exception as e: @@ -30,15 +33,5 @@ def upload(): finally: print(f'{city} - KLAAR met updaten van database') -def truncate(): - try: - sql = "DELETE FROM `parking_cities` WHERE `city`=%s" - cursor.execute(sql, city) - connection.commit() - except Exception as e: - print(f'MySQL error: {e}') - finally: - print(f'{city} - verwijderen van de data') - def update(): return \ No newline at end of file diff --git a/cities/den_haag.py b/cities/den_haag.py index ec7d943..7884a2b 100644 --- a/cities/den_haag.py +++ b/cities/den_haag.py @@ -1,12 +1,12 @@ import json, datetime, uuid -from pymysql import NULL - from database import connection, cursor city = "Den Haag" def upload(): + """Upload the data from the JSON file to the database.""" + den_haag_file = "data/parking-denhaag.json" den_haag_data = open(den_haag_file).read() den_haag_obj = json.loads(den_haag_data) @@ -21,10 +21,11 @@ def upload(): latitude = latitude.replace(']', '') id = uuid.uuid4().hex[:8] + item = item["properties"] sql = """INSERT INTO `parking_cities` (`id`, `city`, `orientation`, `number`, `longitude`, `latitude`, `visibility`, `created_at`, `updated_at`) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""" - val = (id, str(city), str(item["properties"]["ORIENTATIE"]), number(item["properties"]["CAPACITEIT"]), float(longitude), float(latitude), bool(True), (datetime.datetime.now()), (datetime.datetime.now())) + val = (id, str(city), str(item["ORIENTATIE"]), number(item["CAPACITEIT"]), float(longitude), float(latitude), bool(True), (datetime.datetime.now()), (datetime.datetime.now())) cursor.execute(sql, val) connection.commit() except Exception as e: @@ -32,16 +33,6 @@ def upload(): finally: print(f'{city} - KLAAR met updaten van database') -def truncate(): - try: - sql = "DELETE FROM `parking_cities` WHERE `city`=%s" - cursor.execute(sql, city) - connection.commit() - except Exception as e: - print(f'MySQL error: {e}') - finally: - print(f'{city} - verwijderen van de data') - def number(value): if value is None: return 1 diff --git a/cities/zoetermeer.py b/cities/zoetermeer.py index ecb4f02..0f74f3c 100644 --- a/cities/zoetermeer.py +++ b/cities/zoetermeer.py @@ -12,13 +12,17 @@ load_dotenv(dotenv_path=env_path) def download(): + """Download the data as JSON file.""" + # Create a variable and pass the url of file to be downloaded - url = f'{os.getenv("SOURCE")}/77cd514505dd40589f52f99d0e5edd6a_0.geojson' + url = f'{os.getenv("SOURCE")}/308bb3581ba646afad6f776a8f7e4e67_0.geojson' # Copy a network object to a local file - urllib.request.urlretrieve(url, r'data/parking-zoetermeer.json') + urllib.request.urlretrieve(url, 'data/parking-zoetermeer.json') print(f'{city} - KLAAR met downloaden') def upload(): + """Upload the data from the JSON file to the database.""" + zoetermeer_file = "data/parking-zoetermeer.json" zoetermeer_data = open(zoetermeer_file).read() zoetermeer_obj = json.loads(zoetermeer_data) @@ -36,15 +40,5 @@ def upload(): finally: print(f'{city} - KLAAR met updaten van database') -def truncate(): - try: - sql = "DELETE FROM `parking_cities` WHERE `city`=%s" - cursor.execute(sql, city) - connection.commit() - except Exception as e: - print(f'MySQL error: {e}') - finally: - print(f'{city} - verwijderen van de data') - def update(): return \ No newline at end of file diff --git a/database/__init__.py b/database/__init__.py index 86924c7..a4c07f4 100644 --- a/database/__init__.py +++ b/database/__init__.py @@ -17,4 +17,19 @@ # Connect to MySQL connection = pymysql.connect(host=DB_SERVER, port=DB_PORT, user=DB_USERNAME, password=DB_PASSWORD, database=DATABASE) -cursor = connection.cursor() \ No newline at end of file +cursor = connection.cursor() + +def truncate(city): + """Remove all data from a city. + + Args: + city: The name of the city, it's case sensitive! + """ + try: + sql = "DELETE FROM `parking_cities` WHERE `city`=%s" + cursor.execute(sql, city) + connection.commit() + except Exception as e: + print(f'MySQL error: {e}') + finally: + print(f'{city} - verwijderen van de data') \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9cd2ace..d052bcb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,14 @@ +aiohttp==3.7.4.post0 +async-timeout==3.0.1 +attrs==21.2.0 +certifi==2021.10.8 +chardet==4.0.0 +charset-normalizer==2.0.6 +idna==3.2 +multidict==5.2.0 +parking-eindhoven==1.0.0 PyMySQL==1.0.2 python-dotenv==0.19.0 +typing-extensions==3.10.0.2 +urllib3==1.26.7 +yarl==1.7.0