From 08156d1989580450fdec438aad0a6217c374b858 Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 25 Apr 2023 16:46:15 +0200 Subject: [PATCH 1/8] Update url.py allow for login user/password --- erddapy/core/url.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index 4f9ee31..c0b25b2 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import copy import functools import io @@ -48,7 +49,13 @@ def _sort_url(url: str) -> str: def _urlopen(url: str, auth: tuple | None = None, **kwargs: dict) -> BinaryIO: if "timeout" not in kwargs: kwargs["timeout"] = 60 - response = httpx.get(url, follow_redirects=True, auth=auth, **kwargs) + with httpx.Client() as client: + p = parse.urlparse(url) + protocol = 'tabledap' if 'tabledap' in p.path else 'griddap' + login_page = "%s://%s%s/login.html" % (p.scheme, p.netloc, p.path.split('/%s/' % protocol)[0]) + client.post(login_page, data={'user': os.getenv("ERDDAP_USERNAME"), 'password': os.getenv("ERDDAP_PASSWORD")}) + response = client.get(url, follow_redirects=True, auth=auth, **kwargs) + try: response.raise_for_status() except httpx.HTTPError as err: From f6aef256beef3ac7eef508922904eab1cbb03311 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Nov 2023 14:46:05 -0300 Subject: [PATCH 2/8] update pre-commits, drop black, use rev for interrogate --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 33faa56..41d0aaa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: files: requirements-dev.txt - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.7.0 hooks: - id: mypy exclude: docs/source/conf.py From a975990996b8acb9612dcb1c5506cc12988f0991 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Nov 2023 14:48:09 -0300 Subject: [PATCH 3/8] don't vcr simple tests --- tests/cassettes/test_urlopen_raise.yaml | 318 ------------------------ tests/test_url_handling.py | 1 - 2 files changed, 319 deletions(-) delete mode 100644 tests/cassettes/test_urlopen_raise.yaml diff --git a/tests/cassettes/test_urlopen_raise.yaml b/tests/cassettes/test_urlopen_raise.yaml deleted file mode 100644 index 64917aa..0000000 --- a/tests/cassettes/test_urlopen_raise.yaml +++ /dev/null @@ -1,318 +0,0 @@ -interactions: -- request: - body: '' - headers: - accept: - - '*/*' - accept-encoding: - - gzip, deflate, br - connection: - - keep-alive - host: - - developer.mozilla.org - user-agent: - - python-httpx/0.23.0 - method: GET - uri: https://developer.mozilla.org/en-US/404 - response: - content: "\U0001F937\U0001F3FD\u200D\u2640\uFE0F - Page not found | null
MDN Web Docs
" - headers: - Cache-Control: - - no-store, must-revalidate - Connection: - - keep-alive - Content-Encoding: - - br - Content-Security-Policy: - - default-src 'self'; script-src 'report-sample' 'self' www.google-analytics.com/analytics.js - 'sha256-JEt9Nmc3BP88wxuTZm9aKNu87vEgGmKW1zzy/vb1KPs=' polyfill.io/v3/polyfill.min.js - assets.codepen.io production-assets.codepen.io 'sha256-x6Tv+AdV5e6dcolO0TEo+3BG4H2nG2ACjyG8mz6QCes=' - 'sha256-GA8+DpFnqAM/vwERTpb5zyLUaN5KnOhctfTsqWfhaUA='; script-src-elem 'report-sample' - 'self' www.google-analytics.com/analytics.js 'sha256-JEt9Nmc3BP88wxuTZm9aKNu87vEgGmKW1zzy/vb1KPs=' - polyfill.io/v3/polyfill.min.js assets.codepen.io production-assets.codepen.io - 'sha256-x6Tv+AdV5e6dcolO0TEo+3BG4H2nG2ACjyG8mz6QCes=' 'sha256-GA8+DpFnqAM/vwERTpb5zyLUaN5KnOhctfTsqWfhaUA='; - style-src 'report-sample' 'self' 'unsafe-inline'; object-src 'none'; base-uri - 'self'; connect-src 'self' updates.developer.allizom.org updates.developer.mozilla.org - www.google-analytics.com stats.g.doubleclick.net; font-src 'self'; frame-src - 'self' interactive-examples.mdn.mozilla.net interactive-examples.prod.mdn.mozilla.net - interactive-examples.stage.mdn.mozilla.net mdn.github.io yari-demos.prod.mdn.mozit.cloud - mdn.mozillademos.org yari-demos.stage.mdn.mozit.cloud jsfiddle.net www.youtube-nocookie.com - codepen.io www.surveygizmo.com; img-src 'self' *.githubusercontent.com *.googleusercontent.com - *.gravatar.com mozillausercontent.com firefoxusercontent.com profile.stage.mozaws.net - profile.accounts.firefox.com mdn.mozillademos.org media.prod.mdn.mozit.cloud - media.stage.mdn.mozit.cloud interactive-examples.mdn.mozilla.net interactive-examples.prod.mdn.mozilla.net - interactive-examples.stage.mdn.mozilla.net wikipedia.org www.google-analytics.com - www.gstatic.com; manifest-src 'self'; media-src 'self' archive.org videos.cdn.mozilla.net; - child-src 'self'; worker-src 'self'; - Content-Type: - - text/html; charset=utf-8 - Date: - - Wed, 27 Jul 2022 14:57:55 GMT - ETag: - - W/"eb71930fca1bf0f5a1c58df01562ab83" - Last-Modified: - - Wed, 27 Jul 2022 00:41:56 GMT - Server: - - AmazonS3 - Strict-Transport-Security: - - max-age=63072000 - Transfer-Encoding: - - chunked - Vary: - - Accept-Encoding - Via: - - 1.1 c59efe844fd614ff20a756172908fed0.cloudfront.net (CloudFront) - X-Amz-Cf-Id: - - OI75oo7LYqVH6UtsN-ZM_Npub4Dwab18TTS_VaLnbdwgwRjCJaY0uA== - X-Amz-Cf-Pop: - - GRU1-C1 - X-Cache: - - Error from cloudfront - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - DENY - X-XSS-Protection: - - 1; mode=block - http_version: HTTP/1.1 - status_code: 404 -version: 1 diff --git a/tests/test_url_handling.py b/tests/test_url_handling.py index cb604d8..049da4f 100644 --- a/tests/test_url_handling.py +++ b/tests/test_url_handling.py @@ -18,7 +18,6 @@ def test_urlopen(): @pytest.mark.web() -@pytest.mark.vcr() def test_urlopen_raise(): """Assure that urlopen will raise for bad URLs.""" url = "https://developer.mozilla.org/en-US/404" From 333e86d27001a6896ddb2079bfce03790117a7ec Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Nov 2023 14:48:25 -0300 Subject: [PATCH 4/8] run pre-commits --- erddapy/core/url.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index c0b25b2..2f29a71 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -6,6 +6,7 @@ import copy import functools import io +import os from collections import OrderedDict from typing import TYPE_CHECKING @@ -51,9 +52,16 @@ def _urlopen(url: str, auth: tuple | None = None, **kwargs: dict) -> BinaryIO: kwargs["timeout"] = 60 with httpx.Client() as client: p = parse.urlparse(url) - protocol = 'tabledap' if 'tabledap' in p.path else 'griddap' - login_page = "%s://%s%s/login.html" % (p.scheme, p.netloc, p.path.split('/%s/' % protocol)[0]) - client.post(login_page, data={'user': os.getenv("ERDDAP_USERNAME"), 'password': os.getenv("ERDDAP_PASSWORD")}) + protocol = "tabledap" if "tabledap" in p.path else "griddap" + base = p.path.split(f"/{protocol}/")[0] + login_page = f"{p.scheme}://{p.netloc}{base}/login.html" + client.post( + login_page, + data={ + "user": os.getenv("ERDDAP_USERNAME"), + "password": os.getenv("ERDDAP_PASSWORD"), + }, + ) response = client.get(url, follow_redirects=True, auth=auth, **kwargs) try: From 2baf437592f67f4fc1c1a9ccbb2a38755200c418 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Nov 2023 15:23:37 -0300 Subject: [PATCH 5/8] don't vcr simple tests --- tests/test_url_handling.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_url_handling.py b/tests/test_url_handling.py index 049da4f..571a61b 100644 --- a/tests/test_url_handling.py +++ b/tests/test_url_handling.py @@ -9,7 +9,6 @@ @pytest.mark.web() -@pytest.mark.vcr() def test_urlopen(): """Assure that urlopen is always a BytesIO object.""" url = "https://standards.sensors.ioos.us/erddap/tabledap/" @@ -26,7 +25,6 @@ def test_urlopen_raise(): @pytest.mark.web() -@pytest.mark.vcr() def test_check_url_response(): """Test if a bad request returns HTTPError.""" bad_request = ( From b147d26f24a3f484b824b9479552c744b35e154e Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Nov 2023 16:45:38 -0300 Subject: [PATCH 6/8] don't use env vars --- erddapy/core/url.py | 19 +++++++++++-------- erddapy/erddapy.py | 7 ++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index 2f29a71..f1eaa2d 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -50,18 +50,21 @@ def _sort_url(url: str) -> str: def _urlopen(url: str, auth: tuple | None = None, **kwargs: dict) -> BinaryIO: if "timeout" not in kwargs: kwargs["timeout"] = 60 + user = kwargs.pop("user", None) + password = kwargs.pop("password", None) with httpx.Client() as client: p = parse.urlparse(url) protocol = "tabledap" if "tabledap" in p.path else "griddap" base = p.path.split(f"/{protocol}/")[0] - login_page = f"{p.scheme}://{p.netloc}{base}/login.html" - client.post( - login_page, - data={ - "user": os.getenv("ERDDAP_USERNAME"), - "password": os.getenv("ERDDAP_PASSWORD"), - }, - ) + if user is not None and password is not None: + login_page = f"{p.scheme}://{p.netloc}{base}/login.html" + client.post( + login_page, + data={ + "user": f"{user}", + "password": f"{password}", + }, + ) response = client.get(url, follow_redirects=True, auth=auth, **kwargs) try: diff --git a/erddapy/erddapy.py b/erddapy/erddapy.py index e0d86a0..83ea1e7 100644 --- a/erddapy/erddapy.py +++ b/erddapy/erddapy.py @@ -405,11 +405,8 @@ def to_pandas( response = kw.pop("response", "csvp") distinct = kw.pop("distinct", False) url = self.get_download_url(response=response, distinct=distinct) - return to_pandas( - url, - requests_kwargs=requests_kwargs, - pandas_kwargs=dict(**kw), - ) + requests_kwargs = requests_kwargs if requests_kwargs else self.requests_kwargs + return to_pandas(url, requests_kwargs=requests_kwargs, pandas_kwargs=dict(**kw)) def to_ncCF( # noqa: N802 self: ERDDAP, From c4985e15e42e810f9aa3c1074120fd1194a7e327 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 27 Feb 2024 14:14:40 -0300 Subject: [PATCH 7/8] fix lint --- erddapy/core/url.py | 1 - 1 file changed, 1 deletion(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index f1eaa2d..4a86d43 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -6,7 +6,6 @@ import copy import functools import io -import os from collections import OrderedDict from typing import TYPE_CHECKING From 8903d4973f0ea651db0ebcb5b2366e5b8f3e31a5 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 5 Mar 2024 13:46:47 -0300 Subject: [PATCH 8/8] lint --- erddapy/core/url.py | 1 - erddapy/erddapy.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index 4a86d43..b54fcea 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os import copy import functools import io diff --git a/erddapy/erddapy.py b/erddapy/erddapy.py index 83ea1e7..28be59c 100644 --- a/erddapy/erddapy.py +++ b/erddapy/erddapy.py @@ -405,8 +405,14 @@ def to_pandas( response = kw.pop("response", "csvp") distinct = kw.pop("distinct", False) url = self.get_download_url(response=response, distinct=distinct) - requests_kwargs = requests_kwargs if requests_kwargs else self.requests_kwargs - return to_pandas(url, requests_kwargs=requests_kwargs, pandas_kwargs=dict(**kw)) + requests_kwargs = ( + requests_kwargs if requests_kwargs else self.requests_kwargs + ) + return to_pandas( + url, + requests_kwargs=requests_kwargs, + pandas_kwargs=dict(**kw), + ) def to_ncCF( # noqa: N802 self: ERDDAP,