Merge pull request #304 from tekktrik/dev/pypi-stats

Add weekly PyPI stats to daily report

Author: kattni

adabot/circuitpython_libraries.py

@@ -9,6 +9,7 @@
 import datetime
 import inspect
 import logging
+import os
 import re
 import sys
 import traceback
@@ -176,6 +177,13 @@ def run_library_checks(validators, kw_args, error_depth):
     logger.info("")
     logger.info("State of CircuitPython + Libraries + Blinka")
 
+    logger.info("")
+    logger.info("**This report contains information from the previous seven days.**")
+    logger.info(
+        "**Any changes (PRs merged, etc.) made today are not included in this report.**"
+    )
+    logger.info("")
+
     logger.info("### Overall")
     print_pr_overview(lib_insights, core_insights, blinka_insights)
     print_issue_overview(lib_insights, core_insights, blinka_insights)
@@ -216,6 +224,27 @@ def run_library_checks(validators, kw_args, error_depth):
     logger.info("* Core download stats available at https://circuitpython.org/stats")
 
     logger.info("")
+
+    # Get PyPI stats
+    if os.environ.get("BIGQUERY_PRIVATE_KEY") and os.environ["BIGQUERY_CLIENT_EMAIL"]:
+        pypi_stats = dl_stats.retrieve_pypi_stats(repos)
+        library_pypi_names = [
+            repo["name"].replace("_", "-").lower() for repo in common_funcs.list_repos()
+        ]
+        total_library_pypi_stats = 0
+        library_pypi_stats = [
+            stat for stat in pypi_stats if stat.name in library_pypi_names
+        ]
+        top_library_pypi_stats = library_pypi_stats[:10]
+        total_library_pypi_stats = 0
+        for stat in library_pypi_stats:
+            total_library_pypi_stats += stat.num_downloads
+        blinka_pypi_stats = [
+            stat for stat in pypi_stats if stat.name == "adafruit-blinka"
+        ][0]
+    else:
+        pypi_stats = None
+
     logger.info("### Libraries")
     print_pr_overview(lib_insights)
     logger.info("  * Merged pull requests:")
@@ -247,6 +276,30 @@ def run_library_checks(validators, kw_args, error_depth):
 
     logger.info("* https://circuitpython.org/contributing")
 
+    logger.info("")
+    logger.info("#### Library PyPI Weekly Download Stats")
+    if pypi_stats:
+        logger.info("* **Total Library Stats**")
+        logger.info(
+            "  * %d PyPI downloads over %d libraries",
+            total_library_pypi_stats,
+            len(library_pypi_names),
+        )
+        logger.info("* **Top 10 Libraries by PyPI Downloads**")
+        for lib_stat in top_library_pypi_stats:
+            logger.info(
+                "  * %s: %d",
+                lib_stat.name,
+                lib_stat.num_downloads,
+            )
+    else:
+        logger.info(
+            "*This info is not printed because the cron is unable to access secrets.*"
+        )
+        logger.info(
+            "*This is expected if this was run as part of the CI for a pull request.*"
+        )
+
     logger.info("")
     logger.info("#### Library updates in the last seven days:")
     if len(new_libs) != 0:
@@ -295,7 +348,20 @@ def run_library_checks(validators, kw_args, error_depth):
     print_issue_overview(blinka_insights)
     logger.info("* %s open issues", len(blinka_insights["open_issues"]))
     logger.info("  * https://github.com/adafruit/Adafruit_Blinka/issues")
-    blinka_dl = dl_stats.piwheels_stats().get("adafruit-blinka", {}).get("month", "N/A")
+    blinka_dl = (
+        dl_stats.retrieve_piwheels_stats()
+        .get("adafruit-blinka", {})
+        .get("month", "N/A")
+    )
+    if pypi_stats:
+        logger.info(
+            "* PyPI downloads in the last week: %d",
+            blinka_pypi_stats.num_downloads,
+        )
+    else:
+        logger.info(
+            "* PyPI download stats unavailable - this is normal for CI runs triggered by PRs"
+        )
     logger.info("* Piwheels Downloads in the last month: %s", blinka_dl)
     logger.info("Number of supported boards: %s", blinka_funcs.board_count())
 

adabot/circuitpython_library_download_stats.py

@@ -6,16 +6,23 @@
     and each library.
 """
 
+import os
 import datetime
 import sys
 import argparse
 import traceback
 import operator
 import requests
 
+import github as pygithub
+from google.cloud import bigquery
+import google.oauth2.service_account
+
 from adabot import github_requests as gh_reqs
 from adabot.lib import common_funcs
 
+GH_INTERFACE = pygithub.Github(os.environ.get("ADABOT_GITHUB_ACCESS_TOKEN"))
+
 # Setup ArgumentParser
 cmd_line_parser = argparse.ArgumentParser(
     description="Adabot utility for CircuitPython Library download stats."
@@ -53,7 +60,7 @@
 PIWHEELS_PACKAGES_URL = "https://www.piwheels.org/packages.json"
 
 
-def piwheels_stats():
+def retrieve_piwheels_stats():
     """Get data dump of piwheels download stats"""
     stats = {}
     response = requests.get(PIWHEELS_PACKAGES_URL)
@@ -68,12 +75,12 @@ def piwheels_stats():
     return stats
 
 
-def get_pypi_stats():
+def parse_piwheels_stats():
     """Map piwheels download stats for each repo"""
     successful_stats = {}
     failed_stats = []
     repos = common_funcs.list_repos()
-    dl_stats = piwheels_stats()
+    dl_stats = retrieve_piwheels_stats()
     for repo in repos:
         if repo["owner"]["login"] == "adafruit" and repo["name"].startswith(
             "Adafruit_CircuitPython"
@@ -101,6 +108,55 @@ def get_pypi_stats():
     return successful_stats, failed_stats
 
 
+def retrieve_pypi_stats(repos):
+    """Get data dump of PyPI download stats (for the last 7 days)"""
+    # Create access info dictionary
+    access_info = {
+        "private_key": os.environ["BIGQUERY_PRIVATE_KEY"],
+        "client_email": os.environ["BIGQUERY_CLIENT_EMAIL"],
+        "token_uri": "https://oauth2.googleapis.com/token",
+    }
+
+    # Use credentials to create a BigQuery client object
+    credentials = google.oauth2.service_account.Credentials.from_service_account_info(
+        access_info
+    )
+    client = bigquery.Client("circuitpython-stats", credentials=credentials)
+
+    # Get the list of PyPI package names
+    packages = [repo["name"].replace("_", "-").lower() for repo in repos]
+
+    # Construct the query to use
+    query = """
+        SELECT
+            file.project as name, COUNT(*) AS num_downloads,
+        FROM
+            `bigquery-public-data.pypi.file_downloads`
+        WHERE DATE(timestamp)
+            BETWEEN DATE_TRUNC(DATE_SUB(CURRENT_DATE(), INTERVAL 7 DAY), DAY)
+            AND DATE_TRUNC(DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY), DAY)
+        AND file.project in (
+            """
+    packages_query = ["?" for _ in packages]
+    query_parameters = [
+        bigquery.ScalarQueryParameter(None, "STRING", package) for package in packages
+    ]
+    query += ",".join(packages_query)
+    query += """
+        )
+        GROUP BY file.project
+        ORDER BY num_downloads DESC
+        """
+
+    # Configure and run the query
+    job_config = bigquery.QueryJobConfig(query_parameters=query_parameters)
+    query_job = client.query(
+        query,
+        job_config=job_config,
+    )
+    return query_job.result()
+
+
 def get_bundle_stats(bundle):
     """Returns the download stats for 'bundle'. Uses release tag names to compile download
     stats for the last 7 days. This assumes an Adabot release within that time frame, and
@@ -172,7 +228,7 @@ def run_stat_check():
     ]
     output_handler("Adafruit CircuitPython Library Piwheels downloads:")
     output_handler()
-    pypi_downloads, pypi_failures = get_pypi_stats()
+    pypi_downloads, pypi_failures = parse_piwheels_stats()
     for stat in sorted(
         pypi_downloads.items(), key=operator.itemgetter(1, 1), reverse=True
     ):

requirements.txt

@@ -15,3 +15,5 @@ parse==1.19.0
 GitPython==3.1.27
 PyGithub==1.55
 typing-extensions~=4.0
+google-auth~=2.13
+google-cloud-bigquery~=3.3