From e191c37796512bc5eac0b4d5a457fc0951778ab9 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Wed, 6 Nov 2024 12:54:48 -0500 Subject: [PATCH 1/2] Change CMR DataFormat to "COG" Fixes #40 --- .devcontainer/devcontainer.json | 32 ++++++++++++++ .devcontainer/docker-compose.yaml | 26 +++++++++++ Dockerfile | 2 +- Makefile | 5 ++- README.md | 3 +- hls_vi/generate_metadata.py | 43 +++++++++++-------- setup.py | 2 +- ...-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml | 2 +- ...-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml | 2 +- tox.ini | 3 +- 10 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yaml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..6a67b52 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile +{ + "name": "Existing Dockerfile", + "build": { + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerfile": "../Dockerfile" + }, + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line to run commands after the container is created. + "postCreateCommand": "python3 -m pip install -e .[test]", + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python" + ] + } + } + + // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "devcontainer" +} diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml new file mode 100644 index 0000000..1b9cf2f --- /dev/null +++ b/.devcontainer/docker-compose.yaml @@ -0,0 +1,26 @@ +# See https://github.com/devcontainers/templates/blob/main/src/docker-existing-docker-compose/.devcontainer/docker-compose.yml +version: '3.8' +services: + # Update this to the name of the service you want to work with in your docker-compose.yml file + tox: + # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer + # folder. Note that the path of the Dockerfile and context is relative to the *primary* + # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile" + # array). The sample below assumes your primary file is in the root of your project. + # + # build: + # context: . + # dockerfile: .devcontainer/Dockerfile + + volumes: + # Update this to wherever you want VS Code to mount the folder of your project + - ..:/workspaces:cached + + # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust. + # cap_add: + # - SYS_PTRACE + # security_opt: + # - seccomp:unconfined + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity diff --git a/Dockerfile b/Dockerfile index ff2f0ee..eba06c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,4 +21,4 @@ RUN : \ WORKDIR /hls_vi COPY ./ ./ -CMD ["tox", "-r", "-v"] +ENTRYPOINT [ "tox" ] diff --git a/Makefile b/Makefile index 12123f6..315cbaa 100644 --- a/Makefile +++ b/Makefile @@ -5,4 +5,7 @@ build: docker compose build test: - docker compose run --rm --build tox + docker compose run --rm --build tox -- -v + +test-metadata: + docker compose run --rm --build tox -- -v -k "not test_generate_indices" diff --git a/README.md b/README.md index 22691bc..43a37a5 100644 --- a/README.md +++ b/README.md @@ -38,5 +38,6 @@ where: You can run tests using Docker: ```bash -make test +make test # Run all tests +make test-metadata # Run only CMR and STAC metadata tests ``` diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index 7fdbce9..2a0045f 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone from pathlib import Path -from typing import List, Optional, Tuple +from typing import List, Tuple import rasterio from lxml import etree as ET @@ -83,7 +83,7 @@ def generate_metadata(input_dir: Path, output_dir: Path) -> None: to this directory with the name `HLS-VI.*.cmr.xml`. """ metadata_path = next(input_dir.glob("HLS.*.cmr.xml")) - tree = ET.parse(str(metadata_path)) + tree = ET.parse(str(metadata_path), None) with rasterio.open(next(output_dir.glob("*.tif"))) as vi_tif: tags = vi_tif.tags() @@ -123,7 +123,13 @@ def generate_metadata(input_dir: Path, output_dir: Path) -> None: tree.find("Temporal/RangeDateTime/BeginningDateTime").text = sensing_time_begin tree.find("Temporal/RangeDateTime/EndingDateTime").text = sensing_time_end - with (importlib_resources.files("hls_vi") / "schema" / "Granule.xsd").open() as xsd: + tree.find("DataFormat").text = "COG" + + with ( + importlib_resources.files("hls_vi") + / "schema" + / "Granule.xsd" # pyright: ignore[reportOperatorIssue] + ).open() as xsd: ET.XMLSchema(file=xsd).assertValid(tree) tree.write( @@ -152,21 +158,22 @@ def normalize_additional_attributes(container: ElementBase) -> None: around the `" + "` and (arbitrarily) using the first value as the value of the additional attribute. """ - attrs: List[ElementBase] = container.findall("./AdditionalAttribute", None) - - for attr in attrs: - value_element: Optional[ElementBase] = attr.find(".//Value", None) - value_text: str = value_element.text if value_element is not None else "" - - if value_element is not None: - # Replace the text of the additional attribute with the first value - # obtained by splitting the text on " + ". If the text does not contain - # " + ", the text remains the same. For example, "05.11".split(" + ") is - # simply ["05.11"], so taking the first element simply produces "05.11". - normalized = value_text.split(" + ", 1)[0].strip() - value_element.text = ( - normalized # pyright: ignore[reportAttributeAccessIssue] - ) + attr_els: List[ElementBase] = container.findall("./AdditionalAttribute", None) + + for attr_el in attr_els: + normalize_additional_attribute(attr_el) + + +def normalize_additional_attribute(attr_el: ElementBase) -> None: + values_el: ElementBase = attr_el.find("./Values", None) + + for el in iter(values_el): + # Replace the text of the additional attribute value with the first value + # obtained by splitting the text on " + ". If the text does not contain + # " + ", the text remains the same. For example, "05.11".split(" + ") is + # simply ["05.11"], so taking the first element simply produces "05.11". + # value_el = ET.SubElement(values_el, "Value", attrib=None, nsmap=None) + el.text = el.text.split(" + ", 1)[0].strip() def set_additional_attribute(attrs: ElementBase, name: str, value: str) -> None: diff --git a/setup.py b/setup.py index 969124d..713b4ed 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ ], extras_require={ "test": [ - "black[jupyter]==21.12b0", + "black[jupyter]==22.8.0", # Last version to support Python 3.6 runtime "flake8", "mypy", "pytest", diff --git a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml index 77fbc45..01ba39b 100644 --- a/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml @@ -245,7 +245,7 @@ - Cloud Optimized GeoTIFF (COG) + COG diff --git a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml index e9334b3..9211d51 100644 --- a/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml +++ b/tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml @@ -307,7 +307,7 @@ - Cloud Optimized GeoTIFF (COG) + COG diff --git a/tox.ini b/tox.ini index 0652a76..fe79ee2 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,6 @@ max-complexity = 12 max-line-length = 90 [testenv] -basepython = python3.6 envdir = venv sitepackages = True extras = @@ -21,4 +20,4 @@ extras = commands = flake8 mypy - pytest -vv --doctest-modules + pytest -vv --doctest-modules {posargs} From f0f5ca12a4a69109ea028fd42075a91663890b6f Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Wed, 6 Nov 2024 16:40:21 -0500 Subject: [PATCH 2/2] Remove errant commented out code --- hls_vi/generate_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hls_vi/generate_metadata.py b/hls_vi/generate_metadata.py index 2a0045f..1bd6681 100644 --- a/hls_vi/generate_metadata.py +++ b/hls_vi/generate_metadata.py @@ -172,7 +172,6 @@ def normalize_additional_attribute(attr_el: ElementBase) -> None: # obtained by splitting the text on " + ". If the text does not contain # " + ", the text remains the same. For example, "05.11".split(" + ") is # simply ["05.11"], so taking the first element simply produces "05.11". - # value_el = ET.SubElement(values_el, "Value", attrib=None, nsmap=None) el.text = el.text.split(" + ", 1)[0].strip()