Change CMR DataFormat to "COG" (#42)

* Change CMR DataFormat to "COG"

Fixes #40

.devcontainer/devcontainer.json

@@ -0,0 +1,32 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+	"name": "Existing Dockerfile",
+	"build": {
+		// Sets the run context to one level up instead of the .devcontainer folder.
+		"context": "..",
+		// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+		"dockerfile": "../Dockerfile"
+	},
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Uncomment the next line to run commands after the container is created.
+	"postCreateCommand": "python3 -m pip install -e .[test]",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python"
+			]
+		}
+	}
+
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,26 @@
+# See https://github.com/devcontainers/templates/blob/main/src/docker-existing-docker-compose/.devcontainer/docker-compose.yml
+version: '3.8'
+services:
+  # Update this to the name of the service you want to work with in your docker-compose.yml file
+  tox:
+    # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer
+    # folder. Note that the path of the Dockerfile and context is relative to the *primary*
+    # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
+    # array). The sample below assumes your primary file is in the root of your project.
+    #
+    # build:
+    #   context: .
+    #   dockerfile: .devcontainer/Dockerfile
+
+    volumes:
+      # Update this to wherever you want VS Code to mount the folder of your project
+      - ..:/workspaces:cached
+
+    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
+    # cap_add:
+    #   - SYS_PTRACE
+    # security_opt:
+    #   - seccomp:unconfined
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity

Dockerfile

@@ -21,4 +21,4 @@ RUN : \
 WORKDIR /hls_vi
 COPY ./ ./
 
-CMD ["tox", "-r", "-v"]
+ENTRYPOINT [ "tox" ]

Makefile

@@ -5,4 +5,7 @@ build:
 	docker compose build
 
 test:
-	docker compose run --rm --build tox
+	docker compose run --rm --build tox -- -v
+
+test-metadata:
+	docker compose run --rm --build tox -- -v -k "not test_generate_indices"

README.md

@@ -38,5 +38,6 @@ where:
 You can run tests using Docker:
 
 ```bash
-make test
+make test           # Run all tests
+make test-metadata  # Run only CMR and STAC metadata tests
 ```

hls_vi/generate_metadata.py

@@ -6,7 +6,7 @@
 
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import List, Optional, Tuple
+from typing import List, Tuple
 
 import rasterio
 from lxml import etree as ET
@@ -83,7 +83,7 @@ def generate_metadata(input_dir: Path, output_dir: Path) -> None:
             to this directory with the name `HLS-VI.*.cmr.xml`.
     """
     metadata_path = next(input_dir.glob("HLS.*.cmr.xml"))
-    tree = ET.parse(str(metadata_path))
+    tree = ET.parse(str(metadata_path), None)
 
     with rasterio.open(next(output_dir.glob("*.tif"))) as vi_tif:
         tags = vi_tif.tags()
@@ -123,7 +123,13 @@ def generate_metadata(input_dir: Path, output_dir: Path) -> None:
     tree.find("Temporal/RangeDateTime/BeginningDateTime").text = sensing_time_begin
     tree.find("Temporal/RangeDateTime/EndingDateTime").text = sensing_time_end
 
-    with (importlib_resources.files("hls_vi") / "schema" / "Granule.xsd").open() as xsd:
+    tree.find("DataFormat").text = "COG"
+
+    with (
+        importlib_resources.files("hls_vi")
+        / "schema"
+        / "Granule.xsd"  # pyright: ignore[reportOperatorIssue]
+    ).open() as xsd:
         ET.XMLSchema(file=xsd).assertValid(tree)
 
     tree.write(
@@ -152,21 +158,21 @@ def normalize_additional_attributes(container: ElementBase) -> None:
     around the `" + "` and (arbitrarily) using the first value as the value of the
     additional attribute.
     """
-    attrs: List[ElementBase] = container.findall("./AdditionalAttribute", None)
-
-    for attr in attrs:
-        value_element: Optional[ElementBase] = attr.find(".//Value", None)
-        value_text: str = value_element.text if value_element is not None else ""
-
-        if value_element is not None:
-            # Replace the text of the additional attribute with the first value
-            # obtained by splitting the text on " + ".  If the text does not contain
-            # " + ", the text remains the same.  For example, "05.11".split(" + ") is
-            # simply ["05.11"], so taking the first element simply produces "05.11".
-            normalized = value_text.split(" + ", 1)[0].strip()
-            value_element.text = (
-                normalized  # pyright: ignore[reportAttributeAccessIssue]
-            )
+    attr_els: List[ElementBase] = container.findall("./AdditionalAttribute", None)
+
+    for attr_el in attr_els:
+        normalize_additional_attribute(attr_el)
+
+
+def normalize_additional_attribute(attr_el: ElementBase) -> None:
+    values_el: ElementBase = attr_el.find("./Values", None)
+
+    for el in iter(values_el):
+        # Replace the text of the additional attribute value with the first value
+        # obtained by splitting the text on " + ".  If the text does not contain
+        # " + ", the text remains the same.  For example, "05.11".split(" + ") is
+        # simply ["05.11"], so taking the first element simply produces "05.11".
+        el.text = el.text.split(" + ", 1)[0].strip()
 
 
 def set_additional_attribute(attrs: ElementBase, name: str, value: str) -> None:

setup.py

@@ -19,7 +19,7 @@
     ],
     extras_require={
         "test": [
-            "black[jupyter]==21.12b0",
+            "black[jupyter]==22.8.0",  # Last version to support Python 3.6 runtime
             "flake8",
             "mypy",
             "pytest",

tests/fixtures/HLS-VI.L30.T06WVS.2024120T211159.v2.0.cmr.xml

@@ -245,7 +245,7 @@
   </OnlineAccessURLs>
   <OnlineResources>
   </OnlineResources>
-  <DataFormat>Cloud Optimized GeoTIFF (COG)</DataFormat>
+  <DataFormat>COG</DataFormat>
   <AssociatedBrowseImageUrls>
   </AssociatedBrowseImageUrls>
 </Granule>

tests/fixtures/HLS-VI.S30.T13RCN.2024128T173909.v2.0.cmr.xml

@@ -307,7 +307,7 @@
   </OnlineAccessURLs>
   <OnlineResources>
   </OnlineResources>
-  <DataFormat>Cloud Optimized GeoTIFF (COG)</DataFormat>
+  <DataFormat>COG</DataFormat>
   <AssociatedBrowseImageUrls>
   </AssociatedBrowseImageUrls>
 </Granule>

tox.ini

@@ -13,12 +13,11 @@ max-complexity = 12
 max-line-length = 90
 
 [testenv]
-basepython = python3.6
 envdir = venv
 sitepackages = True
 extras =
   test
 commands =
   flake8
   mypy
-  pytest -vv --doctest-modules
+  pytest -vv --doctest-modules {posargs}