diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 33faa56..41d0aaa 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,7 +14,7 @@ repos:
files: requirements-dev.txt
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.8.0
+ rev: v1.7.0
hooks:
- id: mypy
exclude: docs/source/conf.py
diff --git a/erddapy/core/url.py b/erddapy/core/url.py
index 4f9ee31..b54fcea 100644
--- a/erddapy/core/url.py
+++ b/erddapy/core/url.py
@@ -48,7 +48,23 @@ def _sort_url(url: str) -> str:
def _urlopen(url: str, auth: tuple | None = None, **kwargs: dict) -> BinaryIO:
if "timeout" not in kwargs:
kwargs["timeout"] = 60
- response = httpx.get(url, follow_redirects=True, auth=auth, **kwargs)
+ user = kwargs.pop("user", None)
+ password = kwargs.pop("password", None)
+ with httpx.Client() as client:
+ p = parse.urlparse(url)
+ protocol = "tabledap" if "tabledap" in p.path else "griddap"
+ base = p.path.split(f"/{protocol}/")[0]
+ if user is not None and password is not None:
+ login_page = f"{p.scheme}://{p.netloc}{base}/login.html"
+ client.post(
+ login_page,
+ data={
+ "user": f"{user}",
+ "password": f"{password}",
+ },
+ )
+ response = client.get(url, follow_redirects=True, auth=auth, **kwargs)
+
try:
response.raise_for_status()
except httpx.HTTPError as err:
diff --git a/erddapy/erddapy.py b/erddapy/erddapy.py
index e0d86a0..28be59c 100644
--- a/erddapy/erddapy.py
+++ b/erddapy/erddapy.py
@@ -405,6 +405,9 @@ def to_pandas(
response = kw.pop("response", "csvp")
distinct = kw.pop("distinct", False)
url = self.get_download_url(response=response, distinct=distinct)
+ requests_kwargs = (
+ requests_kwargs if requests_kwargs else self.requests_kwargs
+ )
return to_pandas(
url,
requests_kwargs=requests_kwargs,
diff --git a/tests/cassettes/test_urlopen_raise.yaml b/tests/cassettes/test_urlopen_raise.yaml
deleted file mode 100644
index 64917aa..0000000
--- a/tests/cassettes/test_urlopen_raise.yaml
+++ /dev/null
@@ -1,318 +0,0 @@
-interactions:
-- request:
- body: ''
- headers:
- accept:
- - '*/*'
- accept-encoding:
- - gzip, deflate, br
- connection:
- - keep-alive
- host:
- - developer.mozilla.org
- user-agent:
- - python-httpx/0.23.0
- method: GET
- uri: https://developer.mozilla.org/en-US/404
- response:
- content: "\U0001F937\U0001F3FD\u200D\u2640\uFE0F
- Page not found | null"
- headers:
- Cache-Control:
- - no-store, must-revalidate
- Connection:
- - keep-alive
- Content-Encoding:
- - br
- Content-Security-Policy:
- - default-src 'self'; script-src 'report-sample' 'self' www.google-analytics.com/analytics.js
- 'sha256-JEt9Nmc3BP88wxuTZm9aKNu87vEgGmKW1zzy/vb1KPs=' polyfill.io/v3/polyfill.min.js
- assets.codepen.io production-assets.codepen.io 'sha256-x6Tv+AdV5e6dcolO0TEo+3BG4H2nG2ACjyG8mz6QCes='
- 'sha256-GA8+DpFnqAM/vwERTpb5zyLUaN5KnOhctfTsqWfhaUA='; script-src-elem 'report-sample'
- 'self' www.google-analytics.com/analytics.js 'sha256-JEt9Nmc3BP88wxuTZm9aKNu87vEgGmKW1zzy/vb1KPs='
- polyfill.io/v3/polyfill.min.js assets.codepen.io production-assets.codepen.io
- 'sha256-x6Tv+AdV5e6dcolO0TEo+3BG4H2nG2ACjyG8mz6QCes=' 'sha256-GA8+DpFnqAM/vwERTpb5zyLUaN5KnOhctfTsqWfhaUA=';
- style-src 'report-sample' 'self' 'unsafe-inline'; object-src 'none'; base-uri
- 'self'; connect-src 'self' updates.developer.allizom.org updates.developer.mozilla.org
- www.google-analytics.com stats.g.doubleclick.net; font-src 'self'; frame-src
- 'self' interactive-examples.mdn.mozilla.net interactive-examples.prod.mdn.mozilla.net
- interactive-examples.stage.mdn.mozilla.net mdn.github.io yari-demos.prod.mdn.mozit.cloud
- mdn.mozillademos.org yari-demos.stage.mdn.mozit.cloud jsfiddle.net www.youtube-nocookie.com
- codepen.io www.surveygizmo.com; img-src 'self' *.githubusercontent.com *.googleusercontent.com
- *.gravatar.com mozillausercontent.com firefoxusercontent.com profile.stage.mozaws.net
- profile.accounts.firefox.com mdn.mozillademos.org media.prod.mdn.mozit.cloud
- media.stage.mdn.mozit.cloud interactive-examples.mdn.mozilla.net interactive-examples.prod.mdn.mozilla.net
- interactive-examples.stage.mdn.mozilla.net wikipedia.org www.google-analytics.com
- www.gstatic.com; manifest-src 'self'; media-src 'self' archive.org videos.cdn.mozilla.net;
- child-src 'self'; worker-src 'self';
- Content-Type:
- - text/html; charset=utf-8
- Date:
- - Wed, 27 Jul 2022 14:57:55 GMT
- ETag:
- - W/"eb71930fca1bf0f5a1c58df01562ab83"
- Last-Modified:
- - Wed, 27 Jul 2022 00:41:56 GMT
- Server:
- - AmazonS3
- Strict-Transport-Security:
- - max-age=63072000
- Transfer-Encoding:
- - chunked
- Vary:
- - Accept-Encoding
- Via:
- - 1.1 c59efe844fd614ff20a756172908fed0.cloudfront.net (CloudFront)
- X-Amz-Cf-Id:
- - OI75oo7LYqVH6UtsN-ZM_Npub4Dwab18TTS_VaLnbdwgwRjCJaY0uA==
- X-Amz-Cf-Pop:
- - GRU1-C1
- X-Cache:
- - Error from cloudfront
- X-Content-Type-Options:
- - nosniff
- X-Frame-Options:
- - DENY
- X-XSS-Protection:
- - 1; mode=block
- http_version: HTTP/1.1
- status_code: 404
-version: 1
diff --git a/tests/test_url_handling.py b/tests/test_url_handling.py
index cb604d8..571a61b 100644
--- a/tests/test_url_handling.py
+++ b/tests/test_url_handling.py
@@ -9,7 +9,6 @@
@pytest.mark.web()
-@pytest.mark.vcr()
def test_urlopen():
"""Assure that urlopen is always a BytesIO object."""
url = "https://standards.sensors.ioos.us/erddap/tabledap/"
@@ -18,7 +17,6 @@ def test_urlopen():
@pytest.mark.web()
-@pytest.mark.vcr()
def test_urlopen_raise():
"""Assure that urlopen will raise for bad URLs."""
url = "https://developer.mozilla.org/en-US/404"
@@ -27,7 +25,6 @@ def test_urlopen_raise():
@pytest.mark.web()
-@pytest.mark.vcr()
def test_check_url_response():
"""Test if a bad request returns HTTPError."""
bad_request = (