Support for TMPDIR and Update to v0.3.2

Author: prabhakk-mw

Advanced-Usage.md

@@ -18,9 +18,10 @@ The following table describes all the environment variables that you can set to
 | **MWI_BASE_URL** | string | `"/matlab"` | Set to control the base URL of the app. MWI_BASE_URL should start with `/` or be `empty`. |
 | **MWI_APP_PORT** | integer | `8080` | Specify the port for the HTTP server to listen on. |
 | **MWI_LOG_LEVEL** | string | `"CRITICAL"` | Specify the Python log level to be one of the following `NOTSET`, `DEBUG`, `INFO`, `WARN`, `ERROR`, or `CRITICAL`. For more information on Python log levels, see [Logging Levels](https://docs.python.org/3/library/logging.html#logging-levels) .<br />The default value is `INFO`. |
-| **MWI_LOG_FILE** | string | `"/tmp/logs.txt"` | Specify the full path to the file where you want the logs to be written. |
+| **MWI_LOG_FILE** | string | `"/tmp/logs.txt"` | Specify the full path to the file where you want debug logs from this integration to be written. |
 | **MWI_WEB_LOGGING_ENABLED** | string | `"True"` | Set this value to `"true"` to see additional web server logs. |
 | **MWI_CUSTOM_HTTP_HEADERS** | string  |`'{"Content-Security-Policy": "frame-ancestors *.example.com:*"}'`<br /> OR <br />`"/path/to/your/custom/http-headers.json"` |Specify valid HTTP headers as JSON data in a string format. <br /> Alternatively, specify the full path to the JSON file containing valid HTTP headers instead. These headers are injected into the HTTP response sent to the browser. </br> For  more information, see the [Custom HTTP Headers](#custom-http-headers) section.|
+| **TMPDIR** or **TMP** | string | `"/path/for/MATLAB/to/use/as/tmp"` | Set either one of these variables to control the temporary folder used by MATLAB. `TMPDIR` takes precedence over `TMP` and if neither variable is set, `/tmp` is the default value used by MATLAB. |
 
 ## Usage outside of Jupyter environment
 

jupyter_matlab_proxy/app.py

@@ -83,7 +83,7 @@ async def start_matlab(req):
     state = req.app["state"]
 
     # Start MATLAB
-    await state.start_matlab(restart=True)
+    await state.start_matlab(restart_matlab=True)
 
     return create_status_response(req.app)
 
@@ -119,7 +119,7 @@ async def set_licensing_info(req):
     if state.is_licensed() is True and not isinstance(state.error, LicensingError):
 
         # Start MATLAB
-        await state.start_matlab(restart=True)
+        await state.start_matlab(restart_matlab=True)
 
     return create_status_response(req.app)
 

jupyter_matlab_proxy/app_state.py

@@ -2,25 +2,23 @@
 
 import asyncio
 from jupyter_matlab_proxy import mwi_environment_variables as mwi_env
-import xml.etree.ElementTree as ET
+from jupyter_matlab_proxy import mwi_embedded_connector as mwi_connector
 import os
 import json
 import pty
 import logging
 from datetime import datetime, timezone, timedelta
 from pathlib import Path
-import tempfile
 import socket
 import errno
 from collections import deque
-from .util import mw, mwi_logger, mwi_validators
+from .util import mw, mwi_logger
 from .util.mwi_exceptions import (
     LicensingError,
     InternalError,
     OnlineLicensingError,
     EntitlementError,
     MatlabInstallError,
-    NetworkLicensingError,
     log_error,
 )
 
@@ -33,6 +31,7 @@ def __init__(self, settings):
         self.settings = settings
         self.processes = {"matlab": None, "xvfb": None}
         self.matlab_port = None
+        self.matlab_ready_file = None
         self.licensing = None
         self.tasks = {}
         self.logs = {
@@ -101,9 +100,10 @@ async def init_licensing(self):
         # If NLM connection string is not present, then look for persistent LNU info
         elif self.__get_cached_licensing_file().exists():
             with open(self.__get_cached_licensing_file(), "r") as f:
-                licensing = json.loads(f.read())
                 logger.info("Found cached licensing information...")
                 try:
+                    # Load can throw if the file is empty for some reason.
+                    licensing = json.loads(f.read())
                     if licensing["type"] == "nlm":
                         # Note: Only NLM settings entered in browser were cached.
                         self.licensing = {
@@ -163,7 +163,7 @@ def get_matlab_state(self):
         elif xvfb.returncode is not None:
             return "down"
         # MATLAB processes started and MATLAB Embedded Connector ready file present
-        elif self.settings["matlab_ready_file"].exists():
+        elif self.matlab_ready_file.exists():
             return "up"
         # MATLAB processes started, but MATLAB Embedded Connector not ready
         return "starting"
@@ -315,17 +315,17 @@ def persist_licensing(self):
             with open(cached_licensing_file, "w") as f:
                 f.write(json.dumps(self.licensing))
 
-    def reserve_matlab_port(self):
-        """Reserve a free port for MATLAB Embedded Connector in the allowed range."""
-
+    def get_free_matlab_port(self):
+        """Returns a free port for MATLAB Embedded Connector in the allowed range."""
+        # NOTE It is not guranteed that the port will remain free!
         # FIXME Because of https://github.com/http-party/node-http-proxy/issues/1342 the
         # node application in development mode always uses port 31515 to bypass the
         # reverse proxy. Once this is addressed, remove this special case.
         if (
             mwi_env.is_development_mode_enabled()
             and not mwi_env.is_testing_mode_enabled()
         ):
-            self.matlab_port = 31515
+            return 31515
         else:
 
             # TODO If MATLAB Connector is enhanced to allow any port, then the
@@ -337,18 +337,18 @@ def reserve_matlab_port(self):
                 try:
                     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                     s.bind(("", port))
-                    self.matlab_port = port
                     s.close()
                     break
                 except socket.error as e:
                     if e.errno != errno.EADDRINUSE:
                         raise e
+        return port
 
-    async def start_matlab(self, restart=False):
+    async def start_matlab(self, restart_matlab=False):
         """Start MATLAB."""
 
         # FIXME
-        if self.get_matlab_state() != "down" and restart is False:
+        if self.get_matlab_state() != "down" and restart_matlab is False:
             raise Exception("MATLAB already running/starting!")
 
         # FIXME
@@ -377,15 +377,14 @@ async def start_matlab(self, restart=False):
         self.logs["matlab"].clear()
 
         # Reserve a port for MATLAB Embedded Connector
-        self.reserve_matlab_port()
+        self.matlab_port = self.get_free_matlab_port()
 
-        # The presence of matlab_ready_file indicates if MATLAB Embedded Connector is
-        # ready to receive connections, but this could be leftover from a terminated
-        # MATLAB, so ensure it is cleaned up before starting MATLAB
-        try:
-            self.settings["matlab_ready_file"].unlink()
-        except FileNotFoundError:
-            pass
+        # Create a folder to hold the matlab_ready_file that will be created by MATLAB to signal readiness
+        self.matlab_ready_file, matlab_log_dir = mwi_connector.get_matlab_ready_file(
+            self.matlab_port
+        )
+        logger.info(f"MATLAB_LOG_DIR:{str(matlab_log_dir)}")
+        logger.info(f"MATLAB_READY_FILE:{str(self.matlab_ready_file)}")
 
         # Configure the environment MATLAB needs to start
         matlab_env = os.environ.copy()
@@ -398,8 +397,8 @@ async def start_matlab(self, restart=False):
             self.settings["matlab_path"] / "ui" / "webgui" / "src"
         )
         matlab_env["MWAPIKEY"] = self.settings["mwapikey"]
-        # TODO Make this configurable (impacts the matlab ready file)
-        matlab_env["MATLAB_LOG_DIR"] = "/tmp"
+        # The matlab ready file is written into this location by MATLAB
+        matlab_env["MATLAB_LOG_DIR"] = str(matlab_log_dir)
         matlab_env["MW_CD_ANYWHERE_ENABLED"] = "true"
         if self.licensing["type"] == "mhlm":
             matlab_env["MLM_WEB_LICENSE"] = "true"
@@ -455,13 +454,10 @@ async def start_matlab(self, restart=False):
         logger.debug(f"Started MATLAB (PID={matlab.pid})")
 
         async def matlab_stderr_reader():
-            logger.info("Starting task to save error logs from MATLAB")
             while not self.processes["matlab"].stderr.at_eof():
-                logger.info("Checking for any error logs from MATLAB to save...")
                 line = await self.processes["matlab"].stderr.readline()
                 if line is None:
                     break
-                logger.info("Saving error logs from MATLAB.")
                 self.logs["matlab"].append(line)
             await self.handle_matlab_output()
 
@@ -488,11 +484,11 @@ async def stop_matlab(self):
 
         # Clean up matlab_ready_file
         try:
-            with open(self.settings["matlab_ready_file"], "r") as mrf:
-                port_in_matlab_ready_file = mrf.read()
-                if str(self.matlab_port) == port_in_matlab_ready_file:
-                    logger.info("Cleaning up matlab_ready_file...")
-                    self.settings["matlab_ready_file"].unlink()
+            if self.matlab_ready_file is not None:
+                logger.info(
+                    f"Cleaning up matlab_ready_file...{str(self.matlab_ready_file)}"
+                )
+                self.matlab_ready_file.unlink()
         except FileNotFoundError:
             # Some other process deleted this file
             pass
@@ -509,15 +505,14 @@ async def handle_matlab_output(self):
         matlab = self.processes["matlab"]
 
         # Wait for MATLAB process to exit
-        logger.info("handle_matlab_output Waiting for MATLAB to exit...")
+        logger.info("Waiting for MATLAB to exit...")
         await matlab.wait()
 
         rc = self.processes["matlab"].returncode
-        logger.info(f"handle_matlab_output MATLAB has exited with errorcode: {rc}")
+        logger.info(f"MATLAB has exited with errorcode: {rc}")
 
         # Look for errors if MATLAB was not intentionally stopped and had an error code
         if len(self.logs["matlab"]) > 0 and self.processes["matlab"].returncode != 0:
-            logger.info(f"handle_matlab_output Some error was found!")
             err = None
             logs = [log.decode().rstrip() for log in self.logs["matlab"]]
 

jupyter_matlab_proxy/devel.py

@@ -1,10 +1,11 @@
 # Copyright 2020-2021 The MathWorks, Inc.
 
 # Development specific functions
-import asyncio, aiohttp
+import asyncio
 from aiohttp import web
 import socket, time, os, sys
 from jupyter_matlab_proxy import mwi_environment_variables as mwi_env
+from jupyter_matlab_proxy import mwi_embedded_connector as mwi_connector
 
 desktop_html = b"""
 <h1>Fake MATLAB Web Desktop</h1>
@@ -79,32 +80,39 @@ async def fake_matlab_started(app):
         print("Diagnostic Information", file=sys.stderr)
         sys.exit(1)
 
-    ready_file = app["ready_file"]
+    # Real MATLAB always uses  $MATLAB_LOG_DIR/connection.securePort as the ready file
+    # We mock reading from the environment variable by calling the helper functions
+    matlab_ready_file, matlab_log_dir = mwi_connector.get_matlab_ready_file(app["port"])
+
     ready_delay = app["ready_delay"]
     try:
         await asyncio.sleep(ready_delay)
-        print(f"Creating fake MATLAB Embedded Connector ready file at {ready_file}")
-        ready_file.touch()
+        print(
+            f"Creating fake MATLAB Embedded Connector ready file at {matlab_ready_file}"
+        )
+        matlab_ready_file.touch()
     except asyncio.CancelledError:
         pass
 
 
 async def start_background_tasks(app):
-    app["ready_file_writer"] = asyncio.create_task(fake_matlab_started(app))
+    await fake_matlab_started(app)
 
 
 async def cleanup_background_tasks(app):
-    app["ready_file_writer"].cancel()
-    await app["ready_file_writer"]
+    # Delete ready file on tear down
+    # NOTE MATLAB does not delete this file on shutdown.
+    matlab_ready_file, matlab_log_dir = mwi_connector.get_matlab_ready_file(app["port"])
+    matlab_ready_file.unlink()
 
 
 def matlab(args):
     port = int(os.environ["MW_CONNECTOR_SECURE_PORT"])
     wait_for_port(port)
     print(f"Serving fake MATLAB Embedded Connector at port {port}")
     app = web.Application()
-    app["ready_file"] = args.ready_file
     app["ready_delay"] = args.ready_delay
+    app["port"] = port
 
     app.router.add_route("GET", "/index-jsd-cr.html", web_handler)
 
@@ -129,18 +137,12 @@ def matlab(args):
 
 if __name__ == "__main__":
     import argparse
-    import tempfile
     from pathlib import Path
 
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers(dest="cmd", required=True)
     matlab_parser = subparsers.add_parser("matlab")
-    matlab_parser.add_argument(
-        "--ready-file",
-        default=Path(tempfile.gettempdir()) / "connector.securePort",
-        type=Path,
-    )
-    matlab_parser.add_argument("--ready-delay", default=10, type=int)
+    matlab_parser.add_argument("--ready-delay", default=2, type=int)
     matlab_parser.set_defaults(func=matlab)
     args = parser.parse_args()
     args.func(args)

jupyter_matlab_proxy/mwi_embedded_connector.py

@@ -0,0 +1,28 @@
+# Copyright 2020-2021 The MathWorks, Inc.
+"""Functions to related to embedded connector access and configuration"""
+
+from jupyter_matlab_proxy import mwi_environment_variables as mwi_env
+from jupyter_matlab_proxy import settings as mwi_settings
+from pathlib import Path
+
+# TODO Write tests
+
+
+def get_matlab_ready_file(connector_port):
+    """Returns the name and location of the file that is used by MATLAB
+    embedded connector to signal its readiness to begin serving content"""
+    ready_file_dir = __create_folder_to_hold_matlab_ready_file(connector_port)
+    ready_file = ready_file_dir / "connector.securePort"
+    return ready_file, ready_file_dir
+
+
+def __create_folder_to_hold_matlab_ready_file(connector_port):
+    """MWI creates the location into which the spawned MATLAB connector can create the ready file"""
+
+    if mwi_env.is_development_mode_enabled():
+        return mwi_settings.get_test_temp_dir()
+
+    matlab_tempdir = Path(mwi_settings.get_matlab_tempdir())
+    matlab_ready_file_dir = matlab_tempdir / "MWI" / str(connector_port)
+    matlab_ready_file_dir.mkdir(parents=True, exist_ok=True)
+    return matlab_ready_file_dir

jupyter_matlab_proxy/mwi_environment_variables.py

@@ -72,6 +72,12 @@ def get_env_name_development():
     return "MWI_DEV"
 
 
+def get_env_name_matlab_tempdir():
+    """The environment variables used to control the temp directory used by MATLAB on POSIX systems"""
+    # Order matters, MATLAB checks TMPDIR first and then TMP
+    return ["TMPDIR", "TMP"]
+
+
 def is_development_mode_enabled():
     """Returns true if the app is in development mode."""
     return os.environ.get(get_env_name_development(), "false").lower() == "true"