Use ruff instead of black as formatter (#3656)

conftest.py

@@ -90,15 +90,14 @@ def pytest_addoption(parser):
 
     # report data to feature parity dashboard
     parser.addoption(
-        "--report-run-url", type=str, action="store", default=None, help="URI of the run who produced the report",
+        "--report-run-url", type=str, action="store", default=None, help="URI of the run who produced the report"
     )
     parser.addoption(
-        "--report-environment", type=str, action="store", default=None, help="The environment the test is run under",
+        "--report-environment", type=str, action="store", default=None, help="The environment the test is run under"
     )
 
 
 def pytest_configure(config):
-
     if not config.option.force_dd_trace_debug and os.environ.get("SYSTEM_TESTS_FORCE_DD_TRACE_DEBUG") == "true":
         config.option.force_dd_trace_debug = True
 
@@ -132,7 +131,6 @@ def pytest_configure(config):
 
 # Called at the very begening
 def pytest_sessionstart(session):
-
     # get the terminal to allow logging directly in stdout
     logger.terminal = session.config.pluginmanager.get_plugin("terminalreporter")
 
@@ -148,7 +146,6 @@ def pytest_sessionstart(session):
 
 # called when each test item is collected
 def _collect_item_metadata(item):
-
     result = {
         "details": None,
         "testDeclaration": None,
@@ -202,7 +199,6 @@ def _get_skip_reason_from_marker(marker):
 
 
 def pytest_pycollect_makemodule(module_path, parent):
-
     # As now, declaration only works for tracers at module level
 
     library = context.scenario.library.library
@@ -230,9 +226,7 @@ def pytest_pycollect_makemodule(module_path, parent):
 
 @pytest.hookimpl(tryfirst=True)
 def pytest_pycollect_makeitem(collector, name, obj):
-
     if collector.istestclass(obj, name):
-
         if obj is None:
             message = f"""{collector.nodeid} is not properly collected.
             You may have forgotten to return a value in a decorator like @features"""
@@ -309,7 +303,6 @@ def _item_is_skipped(item):
 
 
 def pytest_collection_finish(session: pytest.Session):
-
     if session.config.option.collectonly:
         return
 
@@ -328,7 +321,6 @@ def pytest_collection_finish(session: pytest.Session):
 
     last_item_file = ""
     for item in session.items:
-
         if _item_is_skipped(item):
             continue
 
@@ -383,15 +375,13 @@ def pytest_runtest_call(item):
 
 @pytest.hookimpl(optionalhook=True)
 def pytest_json_runtest_metadata(item, call):
-
     if call.when != "setup":
         return {}
 
     return _collect_item_metadata(item)
 
 
 def pytest_json_modifyreport(json_report):
-
     try:
         # add usefull data for reporting
         json_report["context"] = context.serialize()
@@ -403,7 +393,6 @@ def pytest_json_modifyreport(json_report):
 
 
 def pytest_sessionfinish(session, exitstatus):
-
     logger.info("Executing pytest_sessionfinish")
 
     context.scenario.close_targets()
@@ -416,14 +405,14 @@ def pytest_sessionfinish(session, exitstatus):
     if context.scenario.is_main_worker:
         with open(f"{context.scenario.host_log_folder}/known_versions.json", "w", encoding="utf-8") as f:
             json.dump(
-                {library: sorted(versions) for library, versions in LibraryVersion.known_versions.items()}, f, indent=2,
+                {library: sorted(versions) for library, versions in LibraryVersion.known_versions.items()}, f, indent=2
             )
 
         data = session.config._json_report.report  # pylint: disable=protected-access
 
         try:
             junit_modifyreport(
-                data, session.config.option.xmlpath, junit_properties=context.scenario.get_junit_properties(),
+                data, session.config.option.xmlpath, junit_properties=context.scenario.get_junit_properties()
             )
 
             export_feature_parity_dashboard(session, data)
@@ -438,7 +427,6 @@ def pytest_sessionfinish(session, exitstatus):
 
 
 def export_feature_parity_dashboard(session, data):
-
     tests = [convert_test_to_feature_parity_model(test) for test in data["tests"]]
 
     result = {

docs/architecture/test_template.py

@@ -15,6 +15,7 @@
 #  - Skip for every library except one
 #       @irrelevant(context.library = "dotnet", reason="only for .NET")
 
+
 # To run an individual test: ./run.sh tests/test_traces.py::Test_Misc::test_main
 class Test_Misc:
     """*ATTENTION*: This is where you summarize the test"""

format.sh

@@ -36,13 +36,6 @@ fi
 
 source venv/bin/activate
 
-echo "Running black formatter..."
-if [ "$COMMAND" == "fix" ]; then
-  black --quiet .
-else
-  black --check --diff .
-fi
-
 echo "Running mypy type checks..."
 if ! mypy --config pyproject.toml; then
   echo "Mypy type checks failed. Please fix the errors above. 💥 💔 💥"
@@ -55,6 +48,13 @@ if ! which ruff > /dev/null; then
   ./build.sh -i runner > /dev/null
 fi
 
+echo "Running ruff formatter..."
+if [ "$COMMAND" == "fix" ]; then
+  ruff format
+else
+  ruff format --check --diff
+fi
+
 if [ "$COMMAND" == "fix" ]; then
   ruff_args="--fix"
 else

manifests/parser/core.py

@@ -11,7 +11,6 @@ def _flatten(base, obj):
     if base.endswith(".py"):
         base += "::"
     for key, value in obj.items():
-
         if isinstance(value, str):
             yield f"{base}{key}", value
         elif isinstance(value, dict):
@@ -22,7 +21,6 @@ def _flatten(base, obj):
 
 
 def _load_file(file):
-
     try:
         with open(file, encoding="utf-8") as f:
             data = yaml.safe_load(f)

pyproject.toml

@@ -5,10 +5,6 @@ version = '0.0.1'
 [tool.setuptools]
 packages = ["tests", "utils", "manifests"]
 
-[tool.black]
-line-length = 120
-exclude = "(venv/|utils/grpc/weblog_pb2_grpc.py|utils/grpc/weblog_pb2.py|parametric/apps)"
-
 [tool.pytest.ini_options]
 addopts = "--json-report --json-report-indent=2 --color=yes --no-header --junitxml=reportJunit.xml -r Xf"
 testpaths = [
@@ -56,11 +52,24 @@ follow_imports = "skip"
 # enable_error_code = ["ignore-without-code"]
 
 [tool.ruff]
-exclude = ["tests/", "utils/build/", "lib-injection/", "manifests/"]
 line-length = 120
 indent-width = 4
 target-version = "py312"
 
+[tool.ruff.format]
+exclude = [
+    "venv/", 
+    "utils/grpc/weblog_pb2_grpc.py", 
+    "utils/grpc/weblog_pb2.py", "parametric/apps", 
+    "lib-injection/build/docker/python/dd-lib-python-init-test-protobuf-old/addressbook_pb2.py"
+]
+quote-style = "double"
+indent-style = "space"
+# skip-magic-trailing-comma = false
+# line-ending = "auto"
+# docstring-code-format = false
+# docstring-code-line-length = "dynamic"
+
 [tool.ruff.lint]
 select = ["ALL"]
 
@@ -79,7 +88,6 @@ ignore = [
     "ARG005",  # unused lambda argument
     "BLE001",  # TBD
     "C901",    # code complexity, TBD
-    "COM819",  # no extra comma, TBD
     "D200",
     "D205",    # docstring format
     "D401",    # docstring format
@@ -172,6 +180,10 @@ ignore = [
 
 # TODO : remove those ignores
 [tool.ruff.lint.per-file-ignores]
+"tests/*" = ["ALL"]
+"utils/build/*"  = ["ALL"]
+"lib-injection/*" = ["ALL"]
+"manifests/*"  = ["ALL"]
 "utils/grpc/weblog_pb2_grpc.py" = ["ALL"]   # keep this one, it's a generated file
 "utils/grpc/weblog_pb2.py" = ["ALL"]        # keep this one, it's a generated file
 "utils/scripts/*" = ["T201"]                # allow print statements in scripts folder

requirements.txt

@@ -1,11 +1,10 @@
-click==7.1.2  # TODO move once black is moved
+click==7.1.2  # TODO update this
 requests==2.28.1
 pytest==7.1.3
 pytest-json-report==1.5.0
 pytest-split==0.9.0
 pytest-xdist==3.2.1
 pluggy==1.0.0
-black==19.10b0  # TODO : move to 22.8.0
 psutil==6.0.0
 python-dateutil==2.8.2
 msgpack==1.0.4

tests/apm_tracing_e2e/test_otel.py

@@ -4,8 +4,7 @@
 @features.otel_api
 @scenarios.apm_tracing_e2e_otel
 class Test_Otel_Span:
-    """This is a test that that exercises the full flow of APM Tracing with the use of Datadog OTel API.
-    """
+    """This is a test that that exercises the full flow of APM Tracing with the use of Datadog OTel API."""
 
     def setup_datadog_otel_span(self):
         self.req = weblog.get(
@@ -50,7 +49,7 @@ def test_datadog_otel_span(self):
 
     def setup_distributed_otel_trace(self):
         self.req = weblog.get(
-            "/e2e_otel_span/mixed_contrib", {"shouldIndex": 1, "parentName": "root-otel-name.dd-resource"},
+            "/e2e_otel_span/mixed_contrib", {"shouldIndex": 1, "parentName": "root-otel-name.dd-resource"}
         )
 
     @irrelevant(condition=context.library != "golang", reason="Golang specific test with OTel Go contrib package")

tests/appsec/api_security/test_apisec_sampling.py

@@ -38,7 +38,7 @@ def setup_sampling_rate(self):
             weblog.get(
                 f"/tag_value/api_match_AS001/200?{''.join(random.choices(string.ascii_letters, k=16))}={random.randint(1<<31, (1<<32)-1)}"
             )
-            for _ in range(self.N ** 2)
+            for _ in range(self.N**2)
         ]
 
     @irrelevant(
@@ -72,7 +72,6 @@ def setup_sampling_delay(self):
         self.all_requests = [weblog.get("/api_security/sampling/200") for _ in range(10)]
 
     def test_sampling_delay(self):
-
         assert self.request1.status_code == 200
         schema1 = get_schema(self.request1, "req.headers")
         assert schema1 is not None
@@ -98,7 +97,6 @@ def setup_sampling_delay(self):
         self.all_requests = [weblog.get(f"/api_security_sampling/{i}") for i in range(10)]
 
     def test_sampling_delay(self):
-
         assert self.request1.status_code == 200
         schema1 = get_schema(self.request1, "req.headers")
         assert schema1 is not None

tests/appsec/api_security/test_schemas.py

@@ -77,7 +77,7 @@ class Test_Schema_Request_Cookies:
 
     def setup_request_method(self):
         self.request = weblog.get(
-            "/tag_value/api_match_AS001/200", cookies={"secret": "any_value", "cache": "any_other_value"},
+            "/tag_value/api_match_AS001/200", cookies={"secret": "any_value", "cache": "any_other_value"}
         )
 
     @missing_feature(context.library < "[email protected]")
@@ -150,7 +150,7 @@ def test_request_method(self):
         """can provide request request body schema"""
         schema = get_schema(self.request, "req.body")
         assert self.request.status_code == 200
-        assert contains(schema, [{"main": [[[{"key": [8], "value": [16]}]], {"len": 2}], "nullable": [1]}],)
+        assert contains(schema, [{"main": [[[{"key": [8], "value": [16]}]], {"len": 2}], "nullable": [1]}])
 
 
 @rfc("https://docs.google.com/document/d/1OCHPBCAErOL2FhLl64YAHB8woDyq66y5t-JGolxdf1Q/edit#heading=h.bth088vsbjrz")
@@ -176,8 +176,8 @@ def test_request_method(self):
         schema = get_schema(self.request, "req.body")
         assert self.request.status_code == 200
         assert (
-            contains(schema, [{"main": [[[{"key": [8], "value": [8]}]], {"len": 2}], "nullable": [8]}],)
-            or contains(schema, [{"main": [[[{"key": [8], "value": [16]}]], {"len": 2}], "nullable": [1]}],)
+            contains(schema, [{"main": [[[{"key": [8], "value": [8]}]], {"len": 2}], "nullable": [8]}])
+            or contains(schema, [{"main": [[[{"key": [8], "value": [16]}]], {"len": 2}], "nullable": [1]}])
             or contains(
                 schema,
                 [
@@ -221,7 +221,7 @@ class Test_Schema_Response_Body:
     def setup_request_method(self):
         self.request = weblog.post(
             "/tag_value/payload_in_response_body_001/200",
-            data={"test_int": 1, "test_str": "anything", "test_bool": True, "test_float": 1.5234,},
+            data={"test_int": 1, "test_str": "anything", "test_bool": True, "test_float": 1.5234},
         )
 
     def test_request_method(self):
@@ -250,7 +250,7 @@ class Test_Schema_Response_Body_env_var:
     def setup_request_method(self):
         self.request = weblog.post(
             "/tag_value/payload_in_response_body_001/200?X-option=test_value",
-            data={"test_int": 1, "test_str": "anything", "test_bool": True, "test_float": 1.5234,},
+            data={"test_int": 1, "test_str": "anything", "test_bool": True, "test_float": 1.5234},
         )
 
     def test_request_method(self):
@@ -276,8 +276,8 @@ class Test_Scanners:
     def setup_request_method(self):
         self.request = weblog.get(
             "/tag_value/api_match_AS001/200",
-            cookies={"mastercard": "5123456789123456", "authorization": "digest_a0b1c2", "SSN": "123-45-6789",},
-            headers={"authorization": "digest a0b1c2",},
+            cookies={"mastercard": "5123456789123456", "authorization": "digest_a0b1c2", "SSN": "123-45-6789"},
+            headers={"authorization": "digest a0b1c2"},
         )
 
     @missing_feature(context.library < "[email protected]")
@@ -294,7 +294,7 @@ def test_request_method(self):
             {
                 "SSN": [8, {"category": "pii", "type": "us_ssn"}],
                 "authorization": [8],
-                "mastercard": [8, {"card_type": "mastercard", "type": "card", "category": "payment"},],
+                "mastercard": [8, {"card_type": "mastercard", "type": "card", "category": "payment"}],
             },
             {
                 "SSN": [[[8, {"category": "pii", "type": "us_ssn"}]], {"len": 1}],
@@ -311,7 +311,6 @@ def test_request_method(self):
             (schema_cookies[0], EXPECTED_COOKIES),
             (schema_headers[0], EXPECTED_HEADERS),
         ]:
-
             for key in expected[0]:
                 assert key in schema
                 assert isinstance(schema[key], list)

tests/appsec/iast/sink/test_command_injection.py

@@ -41,7 +41,7 @@ def test_telemetry_metric_executed_sink(self):
 )
 @features.iast_stack_trace
 class TestCommandInjection_StackTrace:
-    """Validate stack trace generation """
+    """Validate stack trace generation"""
 
     def setup_stack_trace(self):
         self.r = weblog.post("/iast/cmdi/test_insecure", data={"cmd": "ls"})

tests/appsec/iast/sink/test_hardcoded_passwords.py

@@ -64,7 +64,7 @@ def _get_expectation(self, d):
 )
 @features.iast_stack_trace
 class Test_HardcodedPasswords_StackTrace:
-    """Validate stack trace generation """
+    """Validate stack trace generation"""
 
     def setup_stack_trace(self):
         self.r = weblog.get("/iast/hardcoded_passwords/test_insecure")

tests/appsec/iast/sink/test_hardcoded_secrets.py

@@ -92,7 +92,7 @@ def test_hardcoded_secrets_extended_exec(self):
 )
 @features.iast_stack_trace
 class Test_HardcodedSecrets_StackTrace:
-    """Validate stack trace generation """
+    """Validate stack trace generation"""
 
     def setup_stack_trace(self):
         self.r = weblog.get("/iast/hardcoded_secrets/test_insecure")