Use pathlib instead of os

Author: parthea

describe.py

@@ -28,7 +28,6 @@
 import argparse
 import collections
 import json
-import os
 import pathlib
 import re
 import string
@@ -43,8 +42,9 @@
 
 import uritemplate
 
-DISCOVERY_DOC_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                                    'googleapiclient/discovery_cache/documents')
+DISCOVERY_DOC_DIR = (
+    pathlib.Path(__file__).parent.resolve() / "googleapiclient" / "discovery_cache" / "documents"
+)
 
 CSS = """<style>
 
@@ -136,8 +136,7 @@
   <code><a href="#$name">$name($params)</a></code></p>
 <p class="firstline">$firstline</p>"""
 
-BASE = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                        "docs/dyn")
+BASE = pathlib.Path(__file__).parent.resolve() / "docs" / "dyn"
 
 DIRECTORY_URI = "https://www.googleapis.com/discovery/v1/apis"
 
@@ -361,7 +360,7 @@ def document_collection(resource, path, root_discovery, discovery, css=CSS):
 def document_collection_recursive(resource, path, root_discovery, discovery, doc_destination_dir):
     html = document_collection(resource, path, root_discovery, discovery)
 
-    f = open(os.path.join(doc_destination_dir, path + "html"), "w")
+    f = open(pathlib.Path(doc_destination_dir).joinpath(path + "html"), "w")
 
     f.write(html)
     f.close()
@@ -407,7 +406,7 @@ def document_api(name, version, uri, doc_destination_dir):
         version = safe_version(version)
         doc_name = "{}.{}.json".format(name, version.replace("_", ""))
 
-        discovery_file_path = os.path.join(DISCOVERY_DOC_DIR, doc_name)
+        discovery_file_path = DISCOVERY_DOC_DIR / doc_name
         revision = None
 
         pathlib.Path(discovery_file_path).touch(exist_ok=True)
@@ -495,7 +494,7 @@ def generate_all_api_documents(directory_uri=DIRECTORY_URI, doc_destination_dir=
                 )
             markdown.append("\n")
 
-        with open(os.path.join(BASE, "index.md"), "w") as f:
+        with open(BASE / "index.md", "w") as f:
             markdown = "\n".join(markdown)
             f.write(markdown)
 

noxfile.py

@@ -95,8 +95,8 @@ def unit(session, oauth2client):
 @nox.session(python=["3.9"])
 def scripts(session):
     session.install(*test_dependencies)
-    session.install(".")
-    session.install("-r", os.path.join("scripts", "requirements.txt"))
+    session.install("-e", ".")
+    session.install("-r", "scripts/requirements.txt")
 
     # Run py.test against the unit tests.
     session.run(

scripts/buildprbody.py

@@ -14,9 +14,8 @@
 
 from enum import IntEnum
 import numpy as np
-import os
 import pandas as pd
-
+import pathlib
 
 class ChangeType(IntEnum):
     UNKNOWN = 0
@@ -36,8 +35,8 @@ def get_commit_link(name):
     sha = None
     api_link = ""
 
-    file_path = os.path.join(directory, "{0}.sha".format(name))
-    if os.path.exists(file_path):
+    file_path = pathlib.Path(directory).joinpath("{0}.sha".format(name))
+    if file_path.is_file():
         with open(file_path, "r") as f:
             sha = f.readline().rstrip()
             if sha:
@@ -47,7 +46,7 @@ def get_commit_link(name):
 
 
 if __name__ == "__main__":
-    directory = "temp"
+    directory = pathlib.Path("temp")
     dataframe = pd.read_csv("temp/allapis.dataframe")
     dataframe["Version"] = dataframe["Version"].astype(str)
 
@@ -80,7 +79,7 @@ def get_commit_link(name):
         .values
     )
 
-    with open(os.path.join(directory, "allapis.summary"), "w") as f:
+    with open(directory / "allapis.summary", "w") as f:
         if len(stable_and_breaking) > 0:
             f.writelines(
                 [

scripts/changesummary.py

@@ -15,12 +15,17 @@
 from enum import IntEnum
 import json
 from multiprocessing import Pool
-import os
 import pandas as pd
+import pathlib
 import numpy as np
 
-BRANCH_ARTIFACTS_DIR = "googleapiclient/discovery_cache/documents/"
-MAIN_ARTIFACTS_DIR = "../main/googleapiclient/discovery_cache/documents/"
+BRANCH_ARTIFACTS_DIR = (
+    pathlib.Path(__file__).parent.resolve() / "googleapiclient" / "discovery_cache" / "documents"
+)
+MAIN_ARTIFACTS_DIR = (
+    pathlib.Path(__file__).parent.resolve() / ".." / "main" / "googleapiclient" / "discovery_cache" / "documents"
+)
+
 MULTIPROCESSING_NUM_PER_BATCH = 5
 MULTIPROCESSING_NUM_AGENTS = 10
 
@@ -57,9 +62,9 @@ def __init__(self, new_artifacts_dir, current_artifacts_dir, temp_dir, file_list
         """
 
         self._file_list = file_list
-        self._new_artifacts_dir = new_artifacts_dir
-        self._current_artifacts_dir = current_artifacts_dir
-        self._temp_dir = temp_dir
+        self._new_artifacts_dir = pathlib.Path(new_artifacts_dir)
+        self._current_artifacts_dir = pathlib.Path(current_artifacts_dir)
+        self._temp_dir = pathlib.Path(temp_dir)
 
         # Sanity checks to ensure directories exist
         self._raise_if_directory_not_found(self._new_artifacts_dir)
@@ -73,7 +78,7 @@ def _raise_if_directory_not_found(self, directory):
             directory (str): The relative path to the `directory`
         """
 
-        if not os.path.exists(directory):
+        if not pathlib.Path(directory).exists():
             raise DirectoryDoesNotExist(
                 "Directory does not exist : {0}".format(directory)
             )
@@ -90,7 +95,7 @@ def _load_json_to_dataframe(self, file_path):
         # doesn't exist
         dataframe_doc = pd.DataFrame()
 
-        if os.path.exists(file_path):
+        if pathlib.Path(file_path).is_file():
             with open(file_path, "r") as f:
                 # Now load the json file into a pandas dataframe as a flat table
                 dataframe_doc = pd.json_normalize(json.load(f))
@@ -105,8 +110,8 @@ def _get_discovery_differences(self, filename):
             filename (str): The name of the discovery artifact to parse.
         """
         # The paths of the 2 discovery artifacts to compare
-        current_artifact_path = os.path.join(self._current_artifacts_dir, filename)
-        new_artifact_path = os.path.join(self._new_artifacts_dir, filename)
+        current_artifact_path = self._current_artifacts_dir / filename
+        new_artifact_path = self._new_artifacts_dir / filename
 
         # Use a helper functions to load the discovery artifacts into pandas
         # dataframes
@@ -375,7 +380,7 @@ def _get_summary_and_write_to_disk(self, dataframe, directory):
 
         # Write the final dataframe to disk as it will be used in the
         # buildprbody.py script
-        dataframe.to_csv(os.path.join(directory, "allapis.dataframe"))
+        dataframe.to_csv(directory / "allapis.dataframe")
         return dataframe
 
     def _write_verbose_changes_to_disk(self, dataframe, directory, summary_df):
@@ -432,7 +437,7 @@ def _write_verbose_changes_to_disk(self, dataframe, directory, summary_df):
                 # Create a file which contains verbose changes for the current
                 # API being processed
                 filename = "{0}.verbose".format(currentApi)
-                f = open(os.path.join(directory, filename), "a")
+                f = open(pathlib.Path(directory / filename), "a")
                 lastApi = currentApi
 
                 # Create a filter with only the rows for the current API
@@ -507,7 +512,7 @@ def detect_discovery_changes(self):
 
             # Create a folder which be used by the `createcommits.sh` and
             # `buildprbody.py` scripts.
-            os.makedirs(os.path.dirname(self._temp_dir), exist_ok=True)
+            pathlib.Path(self._temp_dir).mkdir(exist_ok=True)
 
             # Create a summary which contains a conventional commit message
             # for each API and write it to disk.

scripts/changesummary_test.py

@@ -16,7 +16,7 @@
 
 __author__ = "[email protected] (Anthonios Partheniou)"
 
-import os
+import pathlib
 import shutil
 import unittest
 
@@ -26,21 +26,18 @@
 from changesummary import ChangeType
 from changesummary import DirectoryDoesNotExist
 
-
-SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
-NEW_ARTIFACTS_DIR = os.path.join(SCRIPTS_DIR, "test_resources", "new_artifacts_dir")
-CURRENT_ARTIFACTS_DIR = os.path.join(
-    SCRIPTS_DIR, "test_resources", "current_artifacts_dir"
-)
-TEMP_DIR = os.path.join(SCRIPTS_DIR, "test_resources", "temp")
+SCRIPTS_DIR = pathlib.Path(__file__).parent.resolve()
+NEW_ARTIFACTS_DIR = SCRIPTS_DIR / "test_resources" / "new_artifacts_dir"
+CURRENT_ARTIFACTS_DIR = SCRIPTS_DIR / "test_resources" / "current_artifacts_dir"
+TEMP_DIR = SCRIPTS_DIR / "test_resources" / "temp"
 
 
 class TestChangeSummary(unittest.TestCase):
     def setUp(self):
         # Clear temporary directory
         shutil.rmtree(TEMP_DIR, ignore_errors=True)
         # Create temporary directory
-        os.mkdir(TEMP_DIR)
+        pathlib.Path(TEMP_DIR).mkdir()
 
         self.cs = ChangeSummary(NEW_ARTIFACTS_DIR, CURRENT_ARTIFACTS_DIR, TEMP_DIR, [])
 
@@ -66,7 +63,7 @@ def test_raises_on_directory_not_found_temp_dir(self):
             ).detect_discovery_changes()
 
         # Create temporary directory
-        os.mkdir(TEMP_DIR)
+        pathlib.Path(TEMP_DIR).mkdir()
 
         ChangeSummary(
             NEW_ARTIFACTS_DIR, CURRENT_ARTIFACTS_DIR, TEMP_DIR, []
@@ -81,7 +78,7 @@ def test_load_json_to_dataframe_returns_empty_df_if_file_path_invalid(self):
         self.assertTrue(df.empty)
 
     def test_load_json_to_dataframe_returns_expected_data(self):
-        doc_path = os.path.join(NEW_ARTIFACTS_DIR, "drive.v3.json")
+        doc_path = NEW_ARTIFACTS_DIR / "drive.v3.json"
         df = self.cs._load_json_to_dataframe(file_path=doc_path)
         self.assertEqual(df["name"].iloc[0], "drive")
         self.assertEqual(df["version"].iloc[0], "v3")
@@ -184,7 +181,7 @@ def test_detect_discovery_changes(self):
         )
         cs.detect_discovery_changes()
         print("test")
-        result = pd.read_csv(os.path.join(TEMP_DIR, "allapis.dataframe"))
+        result = pd.read_csv(TEMP_DIR / "allapis.dataframe")
 
         # bigquery was added
         # 28 key changes in total.

scripts/updatediscoveryartifacts.py

@@ -12,18 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+import pathlib
+import shutil
 import subprocess
 import tempfile
-import shutil
-import os
 
 import describe
 import changesummary
 
-SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
-DISCOVERY_DOC_DIR = os.path.join(SCRIPTS_DIR, '../googleapiclient/discovery_cache/documents')
-REFERENCE_DOC_DIR = os.path.join(SCRIPTS_DIR, '../docs/dyn')
-TEMP_DIR = os.path.join(SCRIPTS_DIR, 'temp')
+
+SCRIPTS_DIR = pathlib.Path(__file__).parent.resolve()
+DISCOVERY_DOC_DIR = SCRIPTS_DIR / ".." / "googleapiclient" / "discovery_cache" / "documents"
+REFERENCE_DOC_DIR = SCRIPTS_DIR / ".." / "docs" / "dyn"
+TEMP_DIR = SCRIPTS_DIR / "temp"
 
 # Clear discovery documents and reference documents directory
 shutil.rmtree(DISCOVERY_DOC_DIR, ignore_errors=True)
@@ -49,22 +51,22 @@
                                       'origin/master',
                                       '--name-only',
                                       '--',
-                                      (os.path.join(DISCOVERY_DOC_DIR, '*.json')),
-                                      (os.path.join(REFERENCE_DOC_DIR, '*.html')),
-                                      (os.path.join(REFERENCE_DOC_DIR, '*.md')),
+                                      DISCOVERY_DOC_DIR / '*.json',
+                                      REFERENCE_DOC_DIR / '*.html',
+                                      REFERENCE_DOC_DIR / '*.md',
                                       ],
                                       universal_newlines=True)
 
     # Create lists of the changed files
-    all_changed_files = [os.path.basename(file_name) for file_name in git_diff_output.split('\n')]
+    all_changed_files = [pathlib.Path(file_name).name for file_name in git_diff_output.split('\n')]
     json_changed_files = [file for file in all_changed_files if file.endswith(".json")]
 
     # Create temporary directory
-    os.mkdir(TEMP_DIR)
+    pathlib.Path(TEMP_DIR).mkdir()
 
     # Analyze the changes in discovery artifacts using the changesummary module
     changesummary.ChangeSummary(DISCOVERY_DOC_DIR, current_discovery_doc_dir, TEMP_DIR, json_changed_files).detect_discovery_changes()
 
     # Write a list of the files changed to a file called `changed files` which will be used in the `createcommits.sh` script.
-    with open(os.path.join(TEMP_DIR, "changed_files"), "w") as f:
+    with open(TEMP_DIR / "changed_files", "w") as f:
         f.writelines('\n'.join(all_changed_files))