[libc++] Replace 'tags' in CSV status pages by inline notes

This patch replaces 'tags' in the CSV status pages by inline notes
that optionally describe more details about the paper/LWG issue.

Tags were not really useful anymore because we have a vastly superior
tagging system via Github issues, and keeping the tags up-to-date
between CSV files and Github is going to be really challenging.

This patch also adds support for encoding custom notes in the CSV
files via Github issues. To encode a note in the CSV file, the
body (initial description) of a Github issue can be edited to contain
the following markers:

    BEGIN-RST-NOTES
    text that will be added as a note in the RST
    END-RST-NOTES

Author: ldionne

libcxx/docs/Helpers/Styles.rst

@@ -17,13 +17,6 @@
             font-style: italic;
          }
         .complete { background-color: #99FF99; }
-        .chrono { background-color: #D8BFD8; }
-        .format { background-color: #FFE4B5; }
-        .ranges { background-color: #7FFFD4; }
-        .spaceship { background-color: #B0E0E6; }
-        .fc { background-color: #8EAF63; white-space:nowrap; }
-        .concurrency { background-color: #909090; white-space:nowrap; }
-        .dr { background-color: #FFFF99; }
       </style>
 
 .. role:: notstarted
@@ -40,21 +33,5 @@
 .. |Partial| replace:: :partial:`Partial`
 .. |Complete| replace:: :complete:`Complete`
 
-.. role:: chrono
-.. role:: format
-.. role:: ranges
-.. role:: spaceship
-.. role:: fc
-.. role:: concurrency
-.. role:: dr
-
-.. |chrono| replace:: :chrono:`chrono`
-.. |format| replace:: :format:`format`
-.. |ranges| replace:: :ranges:`ranges`
-.. |spaceship| replace:: :spaceship:`spaceship`
-.. |flat_containers| replace:: :fc:`flat containers`
-.. |concurrency TS| replace:: :concurrency:`concurrency TS`
-.. |DR| replace:: :dr:`Defect Report`
-
 .. |sect| unicode:: U+00A7
 .. |hellip| unicode:: U+2026

libcxx/utils/synchronize_csv_status_files.py

@@ -8,6 +8,7 @@
 # ===----------------------------------------------------------------------===##
 
 from typing import List, Dict, Tuple, Optional
+import copy
 import csv
 import itertools
 import json
@@ -19,6 +20,25 @@
 # Number of the 'Libc++ Standards Conformance' project on Github
 LIBCXX_CONFORMANCE_PROJECT = '31'
 
+def extract_between_markers(text: str, begin_marker: str, end_marker: str) -> Optional[str]:
+    """
+    Given a string containing special markers, extract everything located beetwen these markers.
+
+    If the beginning marker is not found, None is returned. If the beginning marker is found but
+    there is no end marker, it is an error (this is done to avoid silently accepting inputs that
+    are erroneous by mistake).
+    """
+    start = text.find(begin_marker)
+    if start == -1:
+        return None
+
+    start += len(begin_marker) # skip the marker itself
+    end = text.find(end_marker, start)
+    if end == -1:
+        raise ArgumentError(f"Could not find end marker {end_marker} in: {text[start:]}")
+
+    return text[start:end]
+
 class PaperStatus:
     TODO = 1
     IN_PROGRESS = 2
@@ -60,19 +80,16 @@ def from_csv_entry(entry: str):
         - '|Partial|'
         - '|Complete|'
         - '|Nothing To Do|'
-
-        Note that since we sometimes add additional notes after the status, we only check that the entry
-        starts with the above patterns.
         """
         if entry == '':
             return PaperStatus(PaperStatus.TODO, entry)
-        elif entry.startswith('|In Progress|'):
+        elif entry == '|In Progress|':
             return PaperStatus(PaperStatus.IN_PROGRESS, entry)
-        elif entry.startswith('|Partial|'):
+        elif entry == '|Partial|':
             return PaperStatus(PaperStatus.PARTIAL, entry)
-        elif entry.startswith('|Complete|'):
+        elif entry == '|Complete|':
             return PaperStatus(PaperStatus.DONE, entry)
-        elif entry.startswith('|Nothing To Do|'):
+        elif entry == '|Nothing To Do|':
             return PaperStatus(PaperStatus.NOTHING_TO_DO, entry)
         else:
             raise RuntimeError(f'Unexpected CSV entry for status: {entry}')
@@ -140,10 +157,10 @@ class PaperInfo:
     First version of LLVM in which this paper/issue was resolved.
     """
 
-    labels: Optional[List[str]]
+    notes: Optional[str]
     """
-    List of labels to associate to the issue in the status-tracking table. Supported labels are
-    'format', 'ranges', 'spaceship', 'flat_containers', 'concurrency TS' and 'DR'.
+    Optional plain text string representing notes to associate to the paper.
+    This is used to populate the "Notes" column in the CSV status pages.
     """
 
     original: Optional[object]
@@ -156,14 +173,14 @@ def __init__(self, paper_number: str, paper_name: str,
                        status: PaperStatus,
                        meeting: Optional[str] = None,
                        first_released_version: Optional[str] = None,
-                       labels: Optional[List[str]] = None,
+                       notes: Optional[str] = None,
                        original: Optional[object] = None):
         self.paper_number = paper_number
         self.paper_name = paper_name
         self.status = status
         self.meeting = meeting
         self.first_released_version = first_released_version
-        self.labels = labels
+        self.notes = notes
         self.original = original
 
     def for_printing(self) -> Tuple[str, str, str, str, str, str]:
@@ -173,7 +190,7 @@ def for_printing(self) -> Tuple[str, str, str, str, str, str]:
             self.meeting if self.meeting is not None else '',
             self.status.to_csv_entry(),
             self.first_released_version if self.first_released_version is not None else '',
-            ' '.join(f'|{label}|' for label in self.labels) if self.labels is not None else '',
+            self.notes if self.notes is not None else '',
         )
 
     def __repr__(self) -> str:
@@ -195,7 +212,7 @@ def from_csv_row(row: Tuple[str, str, str, str, str, str]):# -> PaperInfo:
             status=PaperStatus.from_csv_entry(row[3]),
             meeting=row[2] or None,
             first_released_version=row[4] or None,
-            labels=[l.strip('|') for l in row[5].split(' ') if l] or None,
+            notes=row[5] or None,
             original=row,
         )
 
@@ -210,20 +227,45 @@ def from_github_issue(issue: Dict):# -> PaperInfo:
             raise RuntimeError(f"Issue doesn't have a title that we know how to parse: {issue}")
         paper = match.group(1)
 
-        # Handle labels
-        valid_labels = ('format', 'ranges', 'spaceship', 'flat_containers', 'concurrency TS', 'DR')
-        labels = [label for label in issue['labels'] if label in valid_labels]
+        # Extract any notes from the Github issue and populate the RST notes with them
+        issue_description = issue['content']['body']
+        notes = extract_between_markers(issue_description, 'BEGIN-RST-NOTES', 'END-RST-NOTES')
+        notes = notes.strip() if notes is not None else notes
 
         return PaperInfo(
             paper_number=paper,
             paper_name=issue['title'],
             status=PaperStatus.from_github_issue(issue),
             meeting=issue.get('meeting Voted', None),
             first_released_version=None, # TODO
-            labels=labels if labels else None,
+            notes=notes,
             original=issue,
         )
 
+def merge(paper: PaperInfo, gh: PaperInfo) -> PaperInfo:
+    """
+    Merge a paper coming from a CSV row with a corresponding Github-tracked paper.
+
+    If the CSV row has a status that is "less advanced" than the Github issue, simply update the CSV
+    row with the newer status. Otherwise, report an error if they have a different status because
+    something must be wrong.
+
+    In case we don't update the CSV row's status, we still take any updated notes coming
+    from the Github issue.
+    """
+    if paper.status < gh.status:
+        return gh
+    elif paper.status != gh.status:
+        print(f"We found a CSV row and a Github issue with different statuses:\nrow: {paper}\nGithub issue: {gh}")
+        return paper
+    else:
+        # Retain the notes from the Github issue, if any
+        if gh.notes is not None:
+            cp = copy.deepcopy(paper)
+            cp.notes = gh.notes
+            return cp
+        return paper
+
 def load_csv(file: pathlib.Path) -> List[Tuple]:
     rows = []
     with open(file, newline='') as f:
@@ -275,17 +317,7 @@ def sync_csv(rows: List[Tuple], from_github: List[PaperInfo]) -> List[Tuple]:
             results.append(row)
             continue
 
-        gh = tracking[0]
-
-        # If the CSV row has a status that is "less advanced" than the Github issue, simply update the CSV
-        # row with the newer status. Otherwise, report an error if they have a different status because
-        # something must be wrong.
-        if paper.status < gh.status:
-            results.append(gh.for_printing())
-            continue
-        elif paper.status != gh.status:
-            print(f"We found a CSV row and a Github issue with different statuses:\nrow: {row}\nGithub issue: {gh}")
-        results.append(row)
+        results.append(merge(paper, tracking[0]).for_printing())
 
     return results