diff --git a/backend/__init__.py b/backend/__init__.py index 9bfb659508..3a62a82960 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -9,6 +9,8 @@ from flask_restful import Api from flask_sqlalchemy import SQLAlchemy from flask_mail import Mail +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address from backend.config import EnvironmentConfig @@ -35,7 +37,11 @@ def format_url(endpoint): migrate = Migrate() mail = Mail() oauth = OAuth() - +limiter = Limiter( + storage_uri=EnvironmentConfig.REDIS_URI, + key_func=get_remote_address, + headers_enabled=True, +) osm = oauth.remote_app("osm", app_key="OSM_OAUTH_SETTINGS") # Import all models so that they are registered with SQLAlchemy @@ -64,6 +70,7 @@ def create_app(env="backend.config.EnvironmentConfig"): db.init_app(app) migrate.init_app(app, db) mail.init_app(app) + limiter.init_app(app) app.logger.debug("Add root redirect route") @@ -121,9 +128,21 @@ def add_api_endpoints(app): """ Define the routes the API exposes using Flask-Restful. """ - app.logger.debug("Adding routes to API endpoints") - api = Api(app) + rate_limit_error = { + "RateLimitExceeded": { + "SubCode": "RateLimitExceeded", + "message": "You have exceeded the rate limit. Please try again later.", + "status": 429, + }, + "ConnectionError": { + "SubCode": "RedisConnectionError", + "message": "Connection to Redis server refused.", + "status": 500, + }, + } + api = Api(app, errors=rate_limit_error) + app.logger.debug("Adding routes to API endpoints") # Projects API import from backend.api.projects.resources import ( ProjectsRestAPI, diff --git a/backend/api/campaigns/resources.py b/backend/api/campaigns/resources.py index 47bd9af1a9..127fb3165c 100644 --- a/backend/api/campaigns/resources.py +++ b/backend/api/campaigns/resources.py @@ -1,6 +1,7 @@ from flask_restful import Resource, request, current_app from schematics.exceptions import DataError +from backend import limiter, EnvironmentConfig from backend.models.dtos.campaign_dto import CampaignDTO, NewCampaignDTO from backend.services.campaign_service import CampaignService from backend.services.organisation_service import OrganisationService @@ -212,6 +213,11 @@ def delete(self, campaign_id): class CampaignsAllAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + def get(self): """ Get all active campaigns diff --git a/backend/api/projects/actions.py b/backend/api/projects/actions.py index f43152b22f..2ea191cd6c 100644 --- a/backend/api/projects/actions.py +++ b/backend/api/projects/actions.py @@ -3,6 +3,7 @@ from flask_restful import Resource, request, current_app from schematics.exceptions import DataError +from backend import limiter, EnvironmentConfig from backend.models.dtos.message_dto import MessageDTO from backend.models.dtos.grid_dto import GridDTO from backend.services.project_service import ProjectService, NotFound @@ -78,6 +79,11 @@ def post(self, project_id): class ProjectsActionsMessageContributorsAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, project_id): """ @@ -355,6 +361,11 @@ def post(self, project_id): class ProjectActionsIntersectingTilesAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @tm.pm_only() @token_auth.login_required def post(self): diff --git a/backend/api/projects/resources.py b/backend/api/projects/resources.py index 0dbcb2e807..d649a3a4c7 100644 --- a/backend/api/projects/resources.py +++ b/backend/api/projects/resources.py @@ -4,6 +4,8 @@ from flask_restful import Resource, current_app, request from schematics.exceptions import DataError from distutils.util import strtobool + +from backend import limiter, EnvironmentConfig from backend.models.dtos.project_dto import ( DraftProjectDTO, ProjectDTO, @@ -32,6 +34,11 @@ class ProjectsRestAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required(optional=True) def get(self, project_id): """ diff --git a/backend/api/projects/statistics.py b/backend/api/projects/statistics.py index 566291d050..05e4682eee 100644 --- a/backend/api/projects/statistics.py +++ b/backend/api/projects/statistics.py @@ -1,4 +1,6 @@ from flask_restful import Resource, current_app + +from backend import limiter, EnvironmentConfig from backend.services.stats_service import NotFound, StatsService from backend.services.project_service import ProjectService @@ -28,6 +30,11 @@ def get(self): class ProjectsStatisticsAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["GET"]) + ] + def get(self, project_id): """ Get Project Stats diff --git a/backend/api/system/authentication.py b/backend/api/system/authentication.py index 6d01f22aef..0907ff59ba 100644 --- a/backend/api/system/authentication.py +++ b/backend/api/system/authentication.py @@ -1,7 +1,7 @@ from flask import session, current_app, redirect, request from flask_restful import Resource -from backend import osm +from backend import osm, limiter, EnvironmentConfig from backend.services.users.authentication_service import ( AuthenticationService, AuthServiceError, @@ -17,6 +17,11 @@ def get_oauth_token(): class SystemAuthenticationLoginAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["GET"]) + ] + def get(self): """ Redirects user to OSM to authenticate @@ -44,6 +49,11 @@ def get(self): class SystemAuthenticationCallbackAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["GET"]) + ] + def get(self): """ Handles the OSM OAuth callback diff --git a/backend/api/system/general.py b/backend/api/system/general.py index f7c32e3db2..acd9727468 100644 --- a/backend/api/system/general.py +++ b/backend/api/system/general.py @@ -2,6 +2,7 @@ from flask_restful import Resource, request, current_app from flask_swagger import swagger +from backend import limiter, EnvironmentConfig from backend.services.settings_service import SettingsService from backend.services.messaging.smtp_service import SMTPService @@ -184,6 +185,11 @@ def get(self): class SystemContactAdminRestAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + def post(self): """ Send an email to the system admin diff --git a/backend/api/system/image_upload.py b/backend/api/system/image_upload.py index 8aa7dbb808..fa985c5f6a 100644 --- a/backend/api/system/image_upload.py +++ b/backend/api/system/image_upload.py @@ -3,10 +3,16 @@ from flask_restful import Resource, request, current_app +from backend import limiter, EnvironmentConfig from backend.services.users.authentication_service import token_auth class SystemImageUploadRestAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self): """ diff --git a/backend/api/tasks/actions.py b/backend/api/tasks/actions.py index dbd142dee8..bf0cea5755 100644 --- a/backend/api/tasks/actions.py +++ b/backend/api/tasks/actions.py @@ -1,6 +1,7 @@ from flask_restful import Resource, current_app, request from schematics.exceptions import DataError +from backend import limiter, EnvironmentConfig from backend.models.dtos.grid_dto import SplitTaskDTO from backend.models.postgis.utils import NotFound, InvalidGeoJson from backend.services.grid.split_service import SplitService, SplitServiceError @@ -196,6 +197,11 @@ def post(self, project_id, task_id): class TasksActionsMappingUnlockAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, project_id, task_id): """ @@ -519,6 +525,11 @@ def post(self, project_id): class TasksActionsValidationUnlockAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, project_id): """ @@ -599,6 +610,11 @@ def post(self, project_id): class TasksActionsMapAllAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, project_id): """ @@ -656,6 +672,11 @@ def post(self, project_id): class TasksActionsValidateAllAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, project_id): """ @@ -713,6 +734,11 @@ def post(self, project_id): class TasksActionsInvalidateAllAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, project_id): """ @@ -770,6 +796,11 @@ def post(self, project_id): class TasksActionsResetBadImageryAllAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, project_id): """ @@ -829,6 +860,11 @@ def post(self, project_id): class TasksActionsResetAllAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, project_id): """ diff --git a/backend/api/teams/actions.py b/backend/api/teams/actions.py index f193d9d4f6..844d9bcf67 100644 --- a/backend/api/teams/actions.py +++ b/backend/api/teams/actions.py @@ -2,6 +2,7 @@ from schematics.exceptions import DataError import threading +from backend import limiter, EnvironmentConfig from backend.models.dtos.message_dto import MessageDTO from backend.services.team_service import TeamService, NotFound, TeamJoinNotAllowed from backend.services.users.authentication_service import token_auth, tm @@ -252,6 +253,11 @@ def post(self, team_id): class TeamsActionsMessageMembersAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + @token_auth.login_required def post(self, team_id): """ diff --git a/backend/api/users/actions.py b/backend/api/users/actions.py index 45a108d68a..1fee031bbe 100644 --- a/backend/api/users/actions.py +++ b/backend/api/users/actions.py @@ -1,6 +1,7 @@ from flask_restful import Resource, current_app, request from schematics.exceptions import DataError +from backend import limiter, EnvironmentConfig from backend.models.dtos.user_dto import UserDTO, UserRegisterEmailDTO from backend.services.messaging.message_service import MessageService from backend.services.users.authentication_service import token_auth, tm @@ -314,6 +315,11 @@ def patch(self): class UsersActionsRegisterEmailAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["POST"]) + ] + def post(self): """ Registers users without OpenStreetMap account diff --git a/backend/api/users/statistics.py b/backend/api/users/statistics.py index 05a7dedb3a..b40bfb8dd4 100644 --- a/backend/api/users/statistics.py +++ b/backend/api/users/statistics.py @@ -2,6 +2,7 @@ from datetime import date, timedelta from flask_restful import Resource, request, current_app +from backend import limiter, EnvironmentConfig from backend.services.users.user_service import UserService, NotFound from backend.services.stats_service import StatsService from backend.services.interests_service import InterestService @@ -98,6 +99,11 @@ def get(self, user_id): class UsersStatisticsAllAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["GET"]) + ] + @token_auth.login_required def get(self): """ diff --git a/backend/api/users/tasks.py b/backend/api/users/tasks.py index 792022843a..7563a47de0 100644 --- a/backend/api/users/tasks.py +++ b/backend/api/users/tasks.py @@ -1,11 +1,17 @@ from flask_restful import Resource, current_app, request from dateutil.parser import parse as date_parse +from backend import limiter, EnvironmentConfig from backend.services.users.authentication_service import token_auth from backend.services.users.user_service import UserService, NotFound class UsersTasksAPI(Resource): + + decorators = [ + limiter.limit(EnvironmentConfig.DEFAULT_RATE_LIMIT_THRESHOLD, methods=["GET"]) + ] + @token_auth.login_required def get(self, user_id): """ diff --git a/backend/config.py b/backend/config.py index c30c073268..ab444f5c46 100644 --- a/backend/config.py +++ b/backend/config.py @@ -105,6 +105,18 @@ class EnvironmentConfig: # If disabled project update emails will not be sent. SEND_PROJECT_EMAIL_UPDATES = int(os.getenv("TM_SEND_PROJECT_EMAIL_UPDATES", True)) + # Threshold for rate limiting api calls + DEFAULT_RATE_LIMIT_THRESHOLD = os.getenv( + "TM_API_RATE_LIMIT_THRESHOLD", "100 per hour" + ) + # Memcache configuration + REDIS_PORT = os.getenv("TM_REDIS_PORT", None) + REDIS_HOST = os.getenv("TM_REDIS_HOST", None) + if REDIS_PORT and REDIS_HOST: + REDIS_URI = f"redis://{REDIS_HOST}:{REDIS_PORT}" + else: + REDIS_URI = None + # Languages offered by the Tasking Manager # Please note that there must be exactly the same number of Codes as languages. SUPPORTED_LANGUAGES = { diff --git a/docker-compose.yml b/docker-compose.yml index 1032c4b914..be12439422 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,9 @@ services: <<: *backend container_name: backend restart: always + depends_on: + - postgresql + - redis labels: - traefik.http.routers.backend.rule=Host(`localhost`) && PathPrefix(`/api/`) - traefik.http.services.backend.loadbalancer.server.port=5000 @@ -43,7 +46,14 @@ services: env_file: ${ENV_FILE:-tasking-manager.env} networks: - tm-web - + redis: + image: bitnami/redis:7.0.4 + container_name: redis + environment: + - ALLOW_EMPTY_PASSWORD=yes + restart: always + networks: + - tm-web traefik: image: traefik:v2.3 restart: always diff --git a/example.env b/example.env index 6e8a321143..f08994c203 100644 --- a/example.env +++ b/example.env @@ -149,7 +149,14 @@ POSTGRES_PASSWORD=tm # If disabled project update emails will not be sent. # Set it disabled in case of testing instances -TM_SEND_PROJECT_EMAIL_UPDATES = 1 +TM_SEND_PROJECT_EMAIL_UPDATES=1 + +# Default threshold to rate limit api calls, seperated by comma +TM_API_RATE_LIMIT_THRESHOLD=2/second, 100/hour + +# Redis configuration for rate limiter storage(optional, in memory storage is used by default if not provided.). +TM_REDIS_HOST=localhost +TM_REDIS_PORT=6379 # TM_SERVICE_DESK # If the organisation has a service desk, configures the link diff --git a/requirements.txt b/requirements.txt index df3c30387a..3f338edd24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ Flask-RESTful==0.3.8 Flask-Script==2.0.6 Flask-SQLAlchemy==2.4.4 flask-swagger==0.2.14 +Flask-Limiter==1.5 gevent==20.9.0 GeoAlchemy2==0.8.4 geojson==1.3.4 @@ -35,6 +36,7 @@ Mako==1.1.3 markdown==3.3.3 MarkupSafe==1.1.1 mccabe==0.6.1 +redis==3.5.0 newrelic==5.22.1.152 nose==1.3.7 oauthlib==2.0.2