From 40f6d20f1d0c2fcab12ef7ab9d30439ed1dc79f0 Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 13 Jun 2024 02:07:59 -0700 Subject: [PATCH 1/4] Handle Arduino CLI 1.x `core list` data format The action parses the output of the `arduino-cli core list --format json` command. There was a breaking change to the data format of that output was changed in the 1.0.0 release of Arduino CLI. The action code is updated to handle the new output, while also retaining backwards compatibility with the pre-1.x data format in order to support workflows that pin the Arduino CLI version to >1.0.0 via the action's "cli-version" input. --- compilesketches/compilesketches.py | 22 ++++++++++++++++++- compilesketches/tests/test_compilesketches.py | 15 +++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/compilesketches/compilesketches.py b/compilesketches/compilesketches.py index 23467799..59625648 100644 --- a/compilesketches/compilesketches.py +++ b/compilesketches/compilesketches.py @@ -558,7 +558,7 @@ def __init__(self): self.run_arduino_cli_command(command=["core", "update-index"]) # Use Arduino CLI to get the list of installed platforms command_data = self.run_arduino_cli_command(command=["core", "list", "--format", "json"]) - installed_platform_list = json.loads(command_data.stdout) + installed_platform_list = self.cli_core_list_platform_list(json.loads(command_data.stdout)) for installed_platform in installed_platform_list: if installed_platform[self.cli_json_key("core list", "ID")] == platform[self.dependency_name_key]: # The platform has been installed via Board Manager, so do an overwrite @@ -1437,6 +1437,26 @@ def create_sketches_report_file(self, sketches_report): ) as report_file: json.dump(obj=sketches_report, fp=report_file, indent=2) + def cli_core_list_platform_list(self, data): + """Extract the list of platform data from the `arduino-cli core list` command output according to the Arduino + CLI version in use. + + Keyword arguments: + data -- Arduino CLI command output data + """ + # Interface was changed at this Arduino CLI release: + # https://arduino.github.io/arduino-cli/dev/UPGRADING/#cli-changed-json-output-for-some-lib-core-config-board-and-sketch-commands + first_new_interface_version = "1.0.0" + + if ( + not semver.VersionInfo.is_valid(version=self.cli_version) + or semver.Version.parse(version=self.cli_version).compare(other=first_new_interface_version) >= 0 + ): + # cli_version is either "latest" (which will now always be >=1.0.0) or an explicit version >=1.0.0 + return data["platforms"] + + return data + def cli_json_key(self, command, original_key_name): """Return the appropriate JSON output key name for the Arduino CLI version in use. diff --git a/compilesketches/tests/test_compilesketches.py b/compilesketches/tests/test_compilesketches.py index 8604ecb3..a7d375cb 100644 --- a/compilesketches/tests/test_compilesketches.py +++ b/compilesketches/tests/test_compilesketches.py @@ -2883,6 +2883,21 @@ def test_create_sketches_report_file(monkeypatch, tmp_path): assert json.load(sketch_report_file) == sketches_report +@pytest.mark.parametrize( + "cli_version, data, assertion", + [ + ("latest", {"platforms": [unittest.mock.sentinel.list_item]}, [unittest.mock.sentinel.list_item]), # Non-semver + ("2.0.0", {"platforms": [unittest.mock.sentinel.list_item]}, [unittest.mock.sentinel.list_item]), # > + ("1.0.0", {"platforms": [unittest.mock.sentinel.list_item]}, [unittest.mock.sentinel.list_item]), # == + ("0.1.2", [unittest.mock.sentinel.list_item], [unittest.mock.sentinel.list_item]), # < + ], +) +def test_cli_core_list_platform_list(cli_version, data, assertion): + compile_sketches = get_compilesketches_object(cli_version=cli_version) + + assert compile_sketches.cli_core_list_platform_list(data) == assertion + + @pytest.mark.parametrize( "cli_version, command, original_key, expected_key", [ From 2dc4afd4f44be2a784d912fc068ce70aa4d9861e Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 13 Jun 2024 02:45:49 -0700 Subject: [PATCH 2/4] Remove unused Arduino CLI output key translations The action parses the JSON formatted output of the `arduino-cli core list` command. The action depends on the output having a specific data format, which includes the key names. If the Arduino CLI developers change the name of a key used by the action, it breaks the action. Some time ago, there was a comprehensive review and standardization of the Arduino CLI output key names, which resulted in many changes to the key names. A key name translation system was added to the action code in order to handle that breaking change. Although the action's current code only relies on a couple of keys from a single command, it is planned to use other parts of the output over time (e.g., getting the compilation data now available from the `arduino-cli compile --format json` output instead of parsing the text output as was necessary at the time the action was created). There was an impression of finality to the new key names. For these reasons, the decision was made to implement a comprehensive translation system rather than limiting the scope to only the keys used by the action at the time. Since that time, various key name changes have occurred at various times. Since none of the changed keys were used by the action, this did not cause breakage, but it did render the comprehensive key name translation system outdated. A key used by the action has been renamed once again in the Arduino CLI 1.0.0 release, which requires the translation system to be redesigned and updated. The fact that the comprehensive system has not been kept updated indicates that it is not feasible to maintain such a system. So, rather than investing the significant amount of effort that would be required to review all the changes made to the Arduino CLI output key names and update the comprehensive translation system accordingly, the translation system is trimmed back to only cover the keys actually used by the action. It will be expanded on demand. --- compilesketches/compilesketches.py | 52 ++---------------------------- 1 file changed, 2 insertions(+), 50 deletions(-) diff --git a/compilesketches/compilesketches.py b/compilesketches/compilesketches.py index 59625648..7a3f6f45 100644 --- a/compilesketches/compilesketches.py +++ b/compilesketches/compilesketches.py @@ -1467,58 +1467,10 @@ def cli_json_key(self, command, original_key_name): final_original_interface_version = "0.17.0" # Interface was changed in the next Arduino CLI release key_translation = { - "board details": { - "identification_pref": "identification_prefs", - "usbID": "usb_id", - "PID": "pid", - "VID": "vid", - "websiteURL": "website_url", - "archiveFileName": "archive_filename", - "propertiesId": "properties_id", - "toolsDependencies": "tools_dependencies", - }, - "board list": {"FQBN": "fqbn", "VID": "vid", "PID": "pid"}, - "board listall": { - "FQBN": "fqbn", - "Email": "email", - "ID": "id", - "Installed": "installed", - "Latest": "latest", - "Name": "name", - "Maintainer": "maintainer", - "Website": "website", - }, - "board search": { - "FQBN": "fqbn", - "Email": "email", - "ID": "id", - "Installed": "installed", - "Latest": "latest", - "Name": "name", - "Maintainer": "maintainer", - "Website": "website", - }, "core list": { - "Boards": "boards", - "Email": "email", "ID": "id", - "Installed": "installed", - "Latest": "latest", - "Maintainer": "maintainer", - "Name": "name", - "Website": "website", - }, - "core search": { - "Boards": "boards", - "Email": "email", - "ID": "id", - "Latest": "latest", - "Maintainer": "maintainer", - "Name": "name", - "Website": "website", - }, - "lib deps": {"versionRequired": "version_required", "versionInstalled": "version_installed"}, - "lib search": {"archivefilename": "archive_filename", "cachepath": "cache_path"}, + "Installed": "installed" + } } if ( From a5fd631e786b77472f899018efca480b282cc4d2 Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 13 Jun 2024 05:51:20 -0700 Subject: [PATCH 3/4] Fix position of comment The comment was somehow moved to a different line than the code it refers to. --- compilesketches/tests/test_compilesketches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compilesketches/tests/test_compilesketches.py b/compilesketches/tests/test_compilesketches.py index a7d375cb..ecc1bd7e 100644 --- a/compilesketches/tests/test_compilesketches.py +++ b/compilesketches/tests/test_compilesketches.py @@ -2906,9 +2906,9 @@ def test_cli_core_list_platform_list(cli_version, data, assertion): ("0.17.0", "core list", "ID", "ID"), # == ("0.14.0-rc2", "core list", "ID", "ID"), # < ("1.0.0", "foo", "ID", "ID"), # Command has no translation - ("1.0.0", "core list", "foo", "foo"), + ("1.0.0", "core list", "foo", "foo"), # Key has no translation ], -) # Key has no translation +) def test_cli_json_key(cli_version, command, original_key, expected_key): compile_sketches = get_compilesketches_object(cli_version=cli_version) From 26705b342943dcb94c56a93270d73c1be2e267ab Mon Sep 17 00:00:00 2001 From: per1234 Date: Thu, 13 Jun 2024 03:49:42 -0700 Subject: [PATCH 4/4] Add command output key translation for Arduino CLI 1.0.0 The action parses the output of the `arduino-cli core list --format json` command. The name of one of the keys in that output data used by the action was changed in the 1.0.0 release of Arduino CLI, breaking the action when used with that version of Arduino CLI (which is the default behavior). The action code base already has a key name translation system, which was implemented to handle breakage caused by a previous round of key name changes. However, that system was designed under the assumption that the key names would be stable following the comprehensive redesign that caused the previous changes. It was not designed to handle multiple changes to the name of a given key as has now happened. For this reason, the translation system has been redesigned to support any number of key name changes that might occur. --- compilesketches/compilesketches.py | 50 +++++++++++++------ compilesketches/tests/test_compilesketches.py | 16 +++--- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/compilesketches/compilesketches.py b/compilesketches/compilesketches.py index 7a3f6f45..a02cad5f 100644 --- a/compilesketches/compilesketches.py +++ b/compilesketches/compilesketches.py @@ -560,13 +560,13 @@ def __init__(self): command_data = self.run_arduino_cli_command(command=["core", "list", "--format", "json"]) installed_platform_list = self.cli_core_list_platform_list(json.loads(command_data.stdout)) for installed_platform in installed_platform_list: - if installed_platform[self.cli_json_key("core list", "ID")] == platform[self.dependency_name_key]: + if installed_platform[self.cli_json_key("core list", "id")] == platform[self.dependency_name_key]: # The platform has been installed via Board Manager, so do an overwrite platform_installation_path.path = self.board_manager_platforms_path.joinpath( platform_vendor, "hardware", platform_architecture, - installed_platform[self.cli_json_key("core list", "Installed")], + installed_platform[self.cli_json_key("core list", "installed_version")], ) platform_installation_path.is_overwrite = True @@ -1457,29 +1457,49 @@ def cli_core_list_platform_list(self, data): return data - def cli_json_key(self, command, original_key_name): + def cli_json_key(self, command, key_name): """Return the appropriate JSON output key name for the Arduino CLI version in use. Keyword arguments: command -- Arduino CLI command (e.g., "core list") - original_key_name -- key name used by the original Arduino CLI JSON interface + key_name -- key name used by the current Arduino CLI JSON interface """ - final_original_interface_version = "0.17.0" # Interface was changed in the next Arduino CLI release - - key_translation = { + key_translations = { "core list": { - "ID": "id", - "Installed": "installed" + "id": [ + {"constraints": [">=0.0.0", "<=0.17.0"], "name": "ID"}, + # https://arduino.github.io/arduino-cli/dev/UPGRADING/#arduino-cli-json-output-breaking-changes + {"constraints": [">0.17.0"], "name": "id"}, + ], + "installed_version": [ + {"constraints": [">=0.0.0", "<=0.17.0"], "name": "Installed"}, + # https://arduino.github.io/arduino-cli/dev/UPGRADING/#arduino-cli-json-output-breaking-changes + {"constraints": [">0.17.0", "<1.0.0"], "name": "installed"}, + # https://arduino.github.io/arduino-cli/dev/UPGRADING/#cli-core-list-and-core-search-changed-json-output + {"constraints": [">=1.0.0"], "name": "installed_version"}, + ], } } - if ( - not semver.VersionInfo.is_valid(version=self.cli_version) - or semver.Version.parse(version=self.cli_version).compare(other=final_original_interface_version) > 0 - ) and (command in key_translation and original_key_name in key_translation[command]): - return key_translation[command][original_key_name] + if not semver.VersionInfo.is_valid(version=self.cli_version): + # cli_version is "latest", so use the current key name + return key_name + + for translation in key_translations[command][key_name]: + match = True + for constraint in translation["constraints"]: + if not semver.Version.parse(version=self.cli_version).match(match_expr=constraint): + # The Arduino CLI version does not match the translation's version constraints + match = False + break + + if match: + # The Arduino CLI version matches the translation's version constraints + return translation["name"] - return original_key_name + raise RuntimeError( + f"Translation not implemented for `{key_name}` key of `arduino-cli {command}` for version {self.cli_version}" + ) # pragma: no cover def parse_list_input(list_input): diff --git a/compilesketches/tests/test_compilesketches.py b/compilesketches/tests/test_compilesketches.py index ecc1bd7e..0563ddac 100644 --- a/compilesketches/tests/test_compilesketches.py +++ b/compilesketches/tests/test_compilesketches.py @@ -2899,20 +2899,18 @@ def test_cli_core_list_platform_list(cli_version, data, assertion): @pytest.mark.parametrize( - "cli_version, command, original_key, expected_key", + "cli_version, command, key_name, expected_key", [ - ("latest", "core list", "ID", "id"), # Non-semver - ("1.0.0", "core list", "ID", "id"), # > - ("0.17.0", "core list", "ID", "ID"), # == - ("0.14.0-rc2", "core list", "ID", "ID"), # < - ("1.0.0", "foo", "ID", "ID"), # Command has no translation - ("1.0.0", "core list", "foo", "foo"), # Key has no translation + ("latest", "core list", "installed_version", "installed_version"), # Non-semver + ("0.1.2", "core list", "installed_version", "Installed"), + ("0.17.1", "core list", "installed_version", "installed"), + ("1.2.3", "core list", "installed_version", "installed_version"), ], ) -def test_cli_json_key(cli_version, command, original_key, expected_key): +def test_cli_json_key(cli_version, command, key_name, expected_key): compile_sketches = get_compilesketches_object(cli_version=cli_version) - assert compile_sketches.cli_json_key(command, original_key) == expected_key + assert compile_sketches.cli_json_key(command, key_name) == expected_key @pytest.mark.parametrize("verbose", ["true", "false"])