Document skip_ansible_lint does not work with yamllint rule (#3144)

Author: ssbarnea

.github/workflows/tox.yml

@@ -70,7 +70,7 @@ jobs:
       WSLENV: FORCE_COLOR:PYTEST_REQPASS:TOXENV:GITHUB_STEP_SUMMARY
       # Number of expected test passes, safety measure for accidental skip of
       # tests. Update value if you add/remove tests.
-      PYTEST_REQPASS: 799
+      PYTEST_REQPASS: 801
     steps:
       - name: Activate WSL1
         if: "contains(matrix.shell, 'wsl')"

examples/playbooks/rule-yaml-fail.yml

@@ -0,0 +1,18 @@
+---
+- name: Fixture for yaml rule that should generate 3 errors
+  # https://github.com/ansible/ansible-lint/issues/3139
+  hosts: localhost
+  tasks:
+    - name: "1"
+      ansible.builtin.debug:
+        msg: yes
+
+    - name: "2"
+      ansible.builtin.debug:
+        msg: yes
+      tags:
+        - skip_ansible_lint # this has no effect for yamllint rule
+
+    - name: "3"
+      ansible.builtin.debug:
+        msg: yes

examples/playbooks/rule-yaml-pass.yml

@@ -0,0 +1,7 @@
+---
+# yamllint disable rule:truthy
+- name: Fixture for yaml rule testing ability to use disable comments
+  hosts: localhost
+  tasks: []
+  become: yes # <-- allowed only due to comment above
+# yamllint enable rule:truthy

src/ansiblelint/rules/yaml.md

@@ -28,6 +28,11 @@ warn_list:
   - yaml[document-start]
 ```
 
+!!! warning
+
+    You cannot use `tags: [skip_ansible_lint]` to disable this rule but you can
+    use [yamllint magic comments](https://yamllint.readthedocs.io/en/stable/disable_with_comments.html#disabling-checks-for-all-or-part-of-the-file) for tuning it.
+
 See the
 [list of yamllint rules](https://yamllint.readthedocs.io/en/stable/rules.html)
 for more information.

src/ansiblelint/rules/yaml_rule.py

@@ -38,7 +38,6 @@ class YamllintRule(AnsibleLintRule):
     def matchyaml(self, file: Lintable) -> list[MatchError]:
         """Return matches found for a specific YAML text."""
         matches: list[MatchError] = []
-        filtered_matches: list[MatchError] = []
         if str(file.base_kind) != "text/yaml":
             return matches
 
@@ -58,23 +57,7 @@ def matchyaml(self, file: Lintable) -> list[MatchError]:
                     tag=f"yaml[{problem.rule}]",
                 )
             )
-
-        # Now we save inside the file the skips, so they can be removed later,
-        # especially as these skips can be about other rules than yaml one.
-        _fetch_skips(file.data, file.line_skips)
-
-        for match in matches:
-            last_skips = set()
-
-            for line, skips in file.line_skips.items():
-                if line > match.linenumber:
-                    break
-                last_skips = skips
-            if last_skips.intersection({"skip_ansible_lint", match.rule.id, match.tag}):
-                continue
-            filtered_matches.append(match)
-
-        return filtered_matches
+        return matches
 
 
 def _combine_skip_rules(data: Any) -> set[str]:
@@ -126,37 +109,40 @@ def _fetch_skips(data: Any, collector: dict[int, set[str]]) -> dict[int, set[str
     @pytest.mark.parametrize(
         ("file", "expected_kind", "expected"),
         (
-            (
+            pytest.param(
                 "examples/yamllint/invalid.yml",
                 "yaml",
                 [
                     'Missing document start "---"',
                     'Duplication of key "foo" in mapping',
                     "Trailing spaces",
                 ],
+                id="invalid",
             ),
-            (
-                "examples/yamllint/valid.yml",
-                "yaml",
-                [],
+            pytest.param("examples/yamllint/valid.yml", "yaml", [], id="valid"),
+            pytest.param(
+                "examples/yamllint/multi-document.yaml", "yaml", [], id="multi-document"
             ),
-            (
-                "examples/yamllint/multi-document.yaml",
-                "yaml",
-                [],
+            pytest.param(
+                "examples/yamllint/skipped-rule.yml", "yaml", [], id="skipped-rule"
             ),
-            (
-                "examples/yamllint/skipped-rule.yml",
-                "yaml",
+            pytest.param(
+                "examples/playbooks/rule-yaml-fail.yml",
+                "playbook",
+                [
+                    "Truthy value should be one of [false, true]",
+                    "Truthy value should be one of [false, true]",
+                    "Truthy value should be one of [false, true]",
+                ],
+                id="rule-yaml-fail",
+            ),
+            pytest.param(
+                "examples/playbooks/rule-yaml-pass.yml",
+                "playbook",
                 [],
+                id="rule-yaml-pass",
             ),
         ),
-        ids=(
-            "invalid",
-            "valid",
-            "multi-document",
-            "skipped-rule",
-        ),
     )
     def test_yamllint(file: str, expected_kind: str, expected: list[str]) -> None:
         """Validate parsing of ansible output."""