Merge pull request #219 from righettod/master

Add stats tab (fix  error).

Author: riramar

.github/workflows/tab-stats-headers-generate-related-files.yml

@@ -2,7 +2,7 @@ name: update_tab_stats_related_files
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 0 3 * *'  
+    - cron: '0 0 5 * *'  
   push:
     paths:
       - 'ci/tab_stats_generate_md_file.py'     
@@ -15,15 +15,15 @@ jobs:
       contents: write    
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.10
+    - name: Set up Python
       uses: actions/setup-python@v5
       with:
         python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        sudo apt install -y wget dos2unix
+        sudo apt install -y wget dos2unix sqlite3
     - name: Run update of the tab related files
       run: |
         cd ci; bash tab_stats_manage_generation.sh

assets/tab_stats_generated_images/0753b0c4fecc8c56d81e31f36bc8c397cea5032b.png

@@ -9,16 +9,17 @@
 """
 import sqlite3
 import re
-import requests
+import json
 import hashlib
 from collections import Counter
 from datetime import datetime
 from pathlib import Path
 
 # Constants
-HTTP_REQUEST_TIMEOUT = 60
+DEBUG = True
 DATA_DB_FILE = "/tmp/data.db"
-OSHP_SECURITY_HEADERS_FILE_lOCATION = "https://owasp.org/www-project-secure-headers/ci/headers_add.json"
+OSHP_SECURITY_HEADERS_FILE_lOCATION = "headers_add.json"
+OSHP_SECURITY_HEADERS_EXTRA_FILE_LOCATION = "/tmp/oshp_headers_extra_to_include.txt"
 MD_FILE = "../tab_statistics.md"
 IMAGE_FOLDER_LOCATION = "../assets/tab_stats_generated_images"
 TAB_MD_TEMPLATE = """---
@@ -55,31 +56,51 @@
 # Utility functions
 
 
+def trace(msg):
+    if DEBUG:
+        print(f"[DEBUG] {msg}")
+
+
 def prepare_generation_of_image_from_mermaid(mermaid_code, filename):
+    trace(f"Call prepare_generation_of_image_from_mermaid() => {filename}")
     with open(f"{IMAGE_FOLDER_LOCATION}/{filename}.mmd", "w", encoding="utf-8") as f:
         f.write(mermaid_code + "\n")
+    trace("Call end.")
 
 
 def load_oshp_headers():
+    trace("Call load_oshp_headers()")
     header_names = []
-    resp = requests.get(OSHP_SECURITY_HEADERS_FILE_lOCATION, timeout=HTTP_REQUEST_TIMEOUT)
-    if resp.status_code != 200:
-        raise Exception(f"Status code {resp.status_code} received!")
-    for http_header in resp.json()["headers"]:
-        header_names.append(http_header["name"])
+    trace(f"Call load_oshp_headers() :: Load and parse file {OSHP_SECURITY_HEADERS_FILE_lOCATION}")
+    with open(OSHP_SECURITY_HEADERS_FILE_lOCATION, mode="r", encoding="utf-8") as f:
+        data = json.load(f)
+        http_headers = data["headers"]
+    for http_header in http_headers:
+        header_names.append(http_header["name"].lower())
+    trace(f"Call load_oshp_headers() :: Load file {OSHP_SECURITY_HEADERS_EXTRA_FILE_LOCATION}")
+    with open(OSHP_SECURITY_HEADERS_EXTRA_FILE_LOCATION, mode="r", encoding="utf-8") as f:
+        http_headers = f.read()
+    trace(f"Call load_oshp_headers() :: Parse file {OSHP_SECURITY_HEADERS_EXTRA_FILE_LOCATION}")
+    for http_header in http_headers .split("\n"):
+        header_names.append(http_header.lower().strip(" \n\r\t"))
+    header_names = list(dict.fromkeys(header_names))
     header_names.sort()
+    trace("Call end.")
     return header_names
 
 
 def execute_query_against_data_db(sql_query):
+    trace(f"Call execute_query_against_data_db() => {sql_query}")
     with sqlite3.connect(DATA_DB_FILE) as connection:
         curs = connection.cursor()
         curs.execute(sql_query)
         records = curs.fetchall()
+        trace("Call end.")
         return records
 
 
 def add_stats_section(title, description, chart_mermaid_code):
+    trace(f"Call add_stats_section() => '{title}'")
     with open(MD_FILE, mode="a", encoding="utf-8") as f:
         if chart_mermaid_code is not None and len(chart_mermaid_code.strip()) > 0:
             base_image_filename = hashlib.sha1(title.encode("utf8")).hexdigest()
@@ -88,14 +109,17 @@ def add_stats_section(title, description, chart_mermaid_code):
         else:
             md_code = SECTION_TEMPLATE_NO_MERMAID_CODE % (title, description)
         f.write(f"{md_code}\n")
+    trace("Call end.")
 
 
 def init_stats_file():
+    trace("Call init_stats_file()")
     with open(MD_FILE, mode="w", encoding="utf-8") as f:
         cdate = datetime.now().strftime("%m/%d/%Y at %H:%M:%S")
         f.write(TAB_MD_TEMPLATE)
         f.write("\n\n")
         f.write(f"⏲️ Last update: {cdate} - Domains analyzed count: {get_domains_count()}.\n")
+    trace("Call end.")
 
 
 def get_domains_count():
@@ -248,8 +272,10 @@ def compute_csp_using_directives_with_unsafe_expressions_configuration_global_us
 
 
 if __name__ == "__main__":
+    trace("Clear PNG files")
     for path in Path(IMAGE_FOLDER_LOCATION).glob("*.png"):
         path.unlink()
+    trace("Clear MMD files")
     for path in Path(IMAGE_FOLDER_LOCATION).glob("*.mmd"):
         path.unlink()
     oshp_headers = load_oshp_headers()

assets/tab_stats_generated_images/15d82f7cac9021b254fdf8fed98bb870acc436fb.png

@@ -4,15 +4,20 @@
 #
 # Dependencies:
 #   https://github.com/mermaid-js/mermaid-cli
+#
+# Reference:
+#   https://github.com/mermaid-js/mermaid-cli/blob/master/.github/workflows/test.yml#L24
 #########################################################################
 # Constants
 IMAGE_FOLDER_LOCATION="../assets/tab_stats_generated_images"
 # Generate images
+# We use aa-exec since Ubuntu 24.04's AppArmor profile blocks the use of puppeteer otherwise
+# See https://github.com/puppeteer/puppeteer/issues/12818
 cd $IMAGE_FOLDER_LOCATION
 for mmd_file in *.mmd
 do
     png_file="${mmd_file%%.*}.png"
-    npx -p @mermaid-js/mermaid-cli mmdc --quiet --input $mmd_file --output $png_file --outputFormat png --theme default --backgroundColor transparent
+    aa-exec --profile=chrome npx -p @mermaid-js/mermaid-cli mmdc --quiet --input $mmd_file --output $png_file --outputFormat png --theme default --backgroundColor transparent
 done
 # Only let PNG files
 rm *.mmd

assets/tab_stats_generated_images/2da94599d03c73073ac60b0d8864152f8609cc5b.png

@@ -3,17 +3,35 @@
 # This script manage the generation/update of the tab represented by the 
 # file "tab_statistics.md".
 #########################################################################
+OSHP_SECURITY_HEADERS_EXTRA_FILE_LOCATION="https://raw.githubusercontent.com/oshp/oshp-stats/refs/heads/main/scripts/oshp_headers_extra_to_include.txt"
+OSHP_SECURITY_HEADERS_EXTRA_FILE="/tmp/oshp_headers_extra_to_include.txt"
 DATA_DB_FILE_LOCATION="https://github.com/oshp/oshp-stats/raw/refs/heads/main/data/data.db"
 DATA_DB_FILE="/tmp/data.db"
-echo "[+] Download the database of headers analysis..."
+IMAGE_FOLDER_LOCATION="../assets/tab_stats_generated_images"
+echo "[+] Download the database of headers analysis anc validate the database file..."
 wget -q -O $DATA_DB_FILE $DATA_DB_FILE_LOCATION
+wget -q -O $OSHP_SECURITY_HEADERS_EXTRA_FILE $OSHP_SECURITY_HEADERS_EXTRA_FILE_LOCATION
 file $DATA_DB_FILE
+sqlite3 $DATA_DB_FILE ".tables"
+file $OSHP_SECURITY_HEADERS_EXTRA_FILE
+wc -l $OSHP_SECURITY_HEADERS_EXTRA_FILE
 echo "[+] Set correct access rights for the scripts as well as UNIX CRLF settings..."
-dos2unix tab_stats_generate_*
+dos2unix *.sh
 chmod +x tab_stats_generate_*
 echo "[+] Generate the MD file of the TAB and all the MMD files for every pie chart image..."
 python tab_stats_generate_md_file.py
 echo "[+] Generate the PNG image corresponding to each MMD file..."
 bash tab_stats_generate_png_files.sh
 echo "[+] Cleanup"
-rm $DATA_DB_FILE
+rm $DATA_DB_FILE
+rm $OSHP_SECURITY_HEADERS_EXTRA_FILE
+echo "[+] Check correct generation of the images..."
+img_count=$(find $IMAGE_FOLDER_LOCATION -name "*.png" | wc -l)
+if [ $img_count -eq 0 ]
+then
+    echo "[!] No image file was generated!"
+    exit 1
+else
+    echo "[V] $img_count image files were generated!"
+    exit 0
+fi

assets/tab_stats_generated_images/2ec5e9a684938a169c757a7a631595c53fccc769.png

@@ -17,7 +17,7 @@ tags: headers
 
 
 
-⏲️ Last update: 02/02/2025 at 15:29:30 - Domains analyzed count: 150000.
+⏲️ Last update: 02/02/2025 at 18:10:17 - Domains analyzed count: 150000.
 
 ## Global usage of secure headers
 
@@ -26,88 +26,116 @@ Provide the distribution of usage of secure headers across all domains analyzed.
 ![be611e71c615c27471d766612bfb7e8b05d743c7](assets/tab_stats_generated_images/be611e71c615c27471d766612bfb7e8b05d743c7.png)
 
 
-## Global usage of header 'Cache-Control'
+## Global usage of header 'cache-control'
 
-Provide the distribution of usage of the header 'Cache-Control' across all domains analyzed.
+Provide the distribution of usage of the header 'cache-control' across all domains analyzed.
 
-![5b54b09f5f5c815a014d71b3b07495a69e3a4509](assets/tab_stats_generated_images/5b54b09f5f5c815a014d71b3b07495a69e3a4509.png)
+![577d76c6092c4da6347e1d2c89523dd13a1925f7](assets/tab_stats_generated_images/577d76c6092c4da6347e1d2c89523dd13a1925f7.png)
 
 
-## Global usage of header 'Clear-Site-Data'
+## Global usage of header 'clear-site-data'
 
-Provide the distribution of usage of the header 'Clear-Site-Data' across all domains analyzed.
+Provide the distribution of usage of the header 'clear-site-data' across all domains analyzed.
 
-![2e12376a6c60ad301b25193c11517ea0cd6aba2f](assets/tab_stats_generated_images/2e12376a6c60ad301b25193c11517ea0cd6aba2f.png)
+![49f6a7d15e9a2e3fd4cad94360d37e83ef05fa00](assets/tab_stats_generated_images/49f6a7d15e9a2e3fd4cad94360d37e83ef05fa00.png)
 
 
-## Global usage of header 'Content-Security-Policy'
+## Global usage of header 'content-security-policy'
 
-Provide the distribution of usage of the header 'Content-Security-Policy' across all domains analyzed.
+Provide the distribution of usage of the header 'content-security-policy' across all domains analyzed.
 
-![5e74150e7d98f861bf3fa632ca32e2d7f3e59632](assets/tab_stats_generated_images/5e74150e7d98f861bf3fa632ca32e2d7f3e59632.png)
+![2da94599d03c73073ac60b0d8864152f8609cc5b](assets/tab_stats_generated_images/2da94599d03c73073ac60b0d8864152f8609cc5b.png)
 
 
-## Global usage of header 'Cross-Origin-Embedder-Policy'
+## Global usage of header 'content-security-policy-report-only'
 
-Provide the distribution of usage of the header 'Cross-Origin-Embedder-Policy' across all domains analyzed.
+Provide the distribution of usage of the header 'content-security-policy-report-only' across all domains analyzed.
 
-![00334f25a22543fb684dbe10861afee71c5263e0](assets/tab_stats_generated_images/00334f25a22543fb684dbe10861afee71c5263e0.png)
+![c0b5a705e7e94af3f71ef579bb01b45c2a80ca6b](assets/tab_stats_generated_images/c0b5a705e7e94af3f71ef579bb01b45c2a80ca6b.png)
 
 
-## Global usage of header 'Cross-Origin-Opener-Policy'
+## Global usage of header 'cross-origin-embedder-policy'
 
-Provide the distribution of usage of the header 'Cross-Origin-Opener-Policy' across all domains analyzed.
+Provide the distribution of usage of the header 'cross-origin-embedder-policy' across all domains analyzed.
 
-![f700c02d30083cf617bdeca51e7eec3d49fe4a08](assets/tab_stats_generated_images/f700c02d30083cf617bdeca51e7eec3d49fe4a08.png)
+![0753b0c4fecc8c56d81e31f36bc8c397cea5032b](assets/tab_stats_generated_images/0753b0c4fecc8c56d81e31f36bc8c397cea5032b.png)
 
 
-## Global usage of header 'Cross-Origin-Resource-Policy'
+## Global usage of header 'cross-origin-opener-policy'
 
-Provide the distribution of usage of the header 'Cross-Origin-Resource-Policy' across all domains analyzed.
+Provide the distribution of usage of the header 'cross-origin-opener-policy' across all domains analyzed.
 
-![fa069b07281496f391d957d8936337da1a601614](assets/tab_stats_generated_images/fa069b07281496f391d957d8936337da1a601614.png)
+![e7e550d9cbff786153f7f13f664361e41efee57c](assets/tab_stats_generated_images/e7e550d9cbff786153f7f13f664361e41efee57c.png)
 
 
-## Global usage of header 'Permissions-Policy'
+## Global usage of header 'cross-origin-resource-policy'
 
-Provide the distribution of usage of the header 'Permissions-Policy' across all domains analyzed.
+Provide the distribution of usage of the header 'cross-origin-resource-policy' across all domains analyzed.
 
-![0792b92709f42a7962c27c64b74b94a4dfbffda1](assets/tab_stats_generated_images/0792b92709f42a7962c27c64b74b94a4dfbffda1.png)
+![9cf15b18b743939cbe01342ed5461bc7af6c4d36](assets/tab_stats_generated_images/9cf15b18b743939cbe01342ed5461bc7af6c4d36.png)
 
 
-## Global usage of header 'Referrer-Policy'
+## Global usage of header 'expect-ct'
 
-Provide the distribution of usage of the header 'Referrer-Policy' across all domains analyzed.
+Provide the distribution of usage of the header 'expect-ct' across all domains analyzed.
 
-![d5e855464d800d7b27eb3e430c5ae378497ddf50](assets/tab_stats_generated_images/d5e855464d800d7b27eb3e430c5ae378497ddf50.png)
+![78fc7e8d03077546e27c016ee80b2143dc4ebb08](assets/tab_stats_generated_images/78fc7e8d03077546e27c016ee80b2143dc4ebb08.png)
 
 
-## Global usage of header 'Strict-Transport-Security'
+## Global usage of header 'permissions-policy'
 
-Provide the distribution of usage of the header 'Strict-Transport-Security' across all domains analyzed.
+Provide the distribution of usage of the header 'permissions-policy' across all domains analyzed.
 
-![dbeb94ebb1ed7763f390b7be97a292f3c66920c7](assets/tab_stats_generated_images/dbeb94ebb1ed7763f390b7be97a292f3c66920c7.png)
+![87eabe1fe075f9034dc4db8f76be07da0d08afe3](assets/tab_stats_generated_images/87eabe1fe075f9034dc4db8f76be07da0d08afe3.png)
 
 
-## Global usage of header 'X-Content-Type-Options'
+## Global usage of header 'public-key-pins'
 
-Provide the distribution of usage of the header 'X-Content-Type-Options' across all domains analyzed.
+Provide the distribution of usage of the header 'public-key-pins' across all domains analyzed.
 
-![0259a15512c639e10df724dc019babf03534b303](assets/tab_stats_generated_images/0259a15512c639e10df724dc019babf03534b303.png)
+![e58d592c018472a09777c3fd5440f556bd176dd5](assets/tab_stats_generated_images/e58d592c018472a09777c3fd5440f556bd176dd5.png)
 
 
-## Global usage of header 'X-Frame-Options'
+## Global usage of header 'referrer-policy'
 
-Provide the distribution of usage of the header 'X-Frame-Options' across all domains analyzed.
+Provide the distribution of usage of the header 'referrer-policy' across all domains analyzed.
 
-![6ddd8e89eb34224bf460f672999c7dd447baef79](assets/tab_stats_generated_images/6ddd8e89eb34224bf460f672999c7dd447baef79.png)
+![15d82f7cac9021b254fdf8fed98bb870acc436fb](assets/tab_stats_generated_images/15d82f7cac9021b254fdf8fed98bb870acc436fb.png)
 
 
-## Global usage of header 'X-Permitted-Cross-Domain-Policies'
+## Global usage of header 'strict-transport-security'
 
-Provide the distribution of usage of the header 'X-Permitted-Cross-Domain-Policies' across all domains analyzed.
+Provide the distribution of usage of the header 'strict-transport-security' across all domains analyzed.
 
-![364a633adcd63d315ec3df781fed6008c57ad00d](assets/tab_stats_generated_images/364a633adcd63d315ec3df781fed6008c57ad00d.png)
+![c313c0ceef6eb3116547426b41bdf278df2cc0c6](assets/tab_stats_generated_images/c313c0ceef6eb3116547426b41bdf278df2cc0c6.png)
+
+
+## Global usage of header 'x-content-type-options'
+
+Provide the distribution of usage of the header 'x-content-type-options' across all domains analyzed.
+
+![5808d16f90388bd6309eb12d74010d1c4a8518cf](assets/tab_stats_generated_images/5808d16f90388bd6309eb12d74010d1c4a8518cf.png)
+
+
+## Global usage of header 'x-frame-options'
+
+Provide the distribution of usage of the header 'x-frame-options' across all domains analyzed.
+
+![cfaf56ab8ec6588aa6ee9297b4f93638640d1048](assets/tab_stats_generated_images/cfaf56ab8ec6588aa6ee9297b4f93638640d1048.png)
+
+
+## Global usage of header 'x-permitted-cross-domain-policies'
+
+Provide the distribution of usage of the header 'x-permitted-cross-domain-policies' across all domains analyzed.
+
+![2ec5e9a684938a169c757a7a631595c53fccc769](assets/tab_stats_generated_images/2ec5e9a684938a169c757a7a631595c53fccc769.png)
+
+
+## Global usage of header 'x-xss-protection'
+
+Provide the distribution of usage of the header 'x-xss-protection' across all domains analyzed.
+
+![7b2906800d5eb94d25d0f5cf18322155e8f2192d](assets/tab_stats_generated_images/7b2906800d5eb94d25d0f5cf18322155e8f2192d.png)
 
 
 ## Global usage of insecure framing configuration via the header 'x-frame-options'