Remove python-neutronclient

This patch replaces the deprecated library with openstacksdk. It also
deprecates the use of the [service_auth] section of the configuration
for authentication with Neutron. In a future release these settings
need to be part of the [neutron] configuration section.

Update needed on OVN provider side:
https://review.opendev.org/c/openstack/ovn-octavia-provider/+/870514

Story: 2010509
Task: 47104
Change-Id: I686cfdef78de927fa4bc1921c15e8d5853fd2ef9

Author: weinimo

devstack/plugin.sh

@@ -321,6 +321,16 @@ function octavia_configure {
     iniset $OCTAVIA_CONF service_auth cafile $SSL_BUNDLE_FILE
     iniset $OCTAVIA_CONF service_auth memcached_servers $SERVICE_HOST:11211
 
+    # neutron
+    iniset $OCTAVIA_CONF neutron auth_url $KEYSTONE_SERVICE_URI
+    iniset $OCTAVIA_CONF neutron auth_type password
+    iniset $OCTAVIA_CONF neutron username $OCTAVIA_USERNAME
+    iniset $OCTAVIA_CONF neutron password $OCTAVIA_PASSWORD
+    iniset $OCTAVIA_CONF neutron user_domain_name $OCTAVIA_USER_DOMAIN_NAME
+    iniset $OCTAVIA_CONF neutron project_name $OCTAVIA_PROJECT_NAME
+    iniset $OCTAVIA_CONF neutron project_domain_name $OCTAVIA_PROJECT_DOMAIN_NAME
+    iniset $OCTAVIA_CONF neutron cafile $SSL_BUNDLE_FILE
+
     # Setting other required default options
     iniset $OCTAVIA_CONF controller_worker amphora_driver ${OCTAVIA_AMPHORA_DRIVER}
     iniset $OCTAVIA_CONF controller_worker compute_driver ${OCTAVIA_COMPUTE_DRIVER}

octavia/common/clients.py

@@ -12,9 +12,11 @@
 
 from cinderclient import client as cinder_client
 from glanceclient import client as glance_client
-from neutronclient.neutron import client as neutron_client
+from keystoneauth1 import session
+from keystoneauth1 import token_endpoint
 from novaclient import api_versions
 from novaclient import client as nova_client
+import openstack
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import excutils
@@ -25,7 +27,6 @@
 CONF = cfg.CONF
 
 GLANCE_VERSION = '2'
-NEUTRON_VERSION = '2.0'
 NOVA_VERSION = '2.15'
 CINDER_VERSION = '3'
 
@@ -73,38 +74,20 @@ class NeutronAuth(object):
     neutron_client = None
 
     @classmethod
-    def get_neutron_client(cls, region, service_name=None, endpoint=None,
-                           endpoint_type='publicURL', insecure=False,
-                           ca_cert=None):
-        """Create neutron client object.
-
-        :param region: The region of the service
-        :param service_name: The name of the neutron service in the catalog
-        :param endpoint: The endpoint of the service
-        :param endpoint_type: The endpoint_type of the service
-        :param insecure: Turn off certificate validation
-        :param ca_cert: CA Cert file path
-        :return: a Neutron Client object.
-        :raises Exception: if the client cannot be created
-        """
-        ksession = keystone.KeystoneSession()
+    def get_neutron_client(cls):
+        """Create neutron client object."""
+        ksession = keystone.KeystoneSession('neutron')
         if not cls.neutron_client:
-            kwargs = {'region_name': region,
-                      'session': ksession.get_session(),
-                      'endpoint_type': endpoint_type,
-                      'insecure': insecure}
-            if service_name:
-                kwargs['service_name'] = service_name
-            if endpoint:
-                kwargs['endpoint_override'] = endpoint
-            if ca_cert:
-                kwargs['ca_cert'] = ca_cert
-            try:
-                cls.neutron_client = neutron_client.Client(
-                    NEUTRON_VERSION, **kwargs)
-            except Exception:
-                with excutils.save_and_reraise_exception():
-                    LOG.exception("Error creating Neutron client.")
+            sess = ksession.get_session()
+
+            kwargs = {}
+            if CONF.neutron.endpoint_override:
+                kwargs['network_endpoint_override'] = (
+                    CONF.neutron.endpoint_override)
+
+            conn = openstack.connection.Connection(
+                session=sess, **kwargs)
+            cls.neutron_client = conn
         return cls.neutron_client
 
     @classmethod
@@ -113,26 +96,23 @@ def get_user_neutron_client(cls, context):
 
         It's possible that the token in the context is a trust scoped
         which can't be used to initialize a keystone session.
-
         We directly use the token and endpoint_url to initialize neutron
         client.
         """
-        neutron_endpoint = CONF.neutron.endpoint
-        if not neutron_endpoint:
-            session = keystone.KeystoneSession().get_session()
-            endpoint_data = session.get_endpoint_data(
+        sess = keystone.KeystoneSession('neutron').get_session()
+        neutron_endpoint = CONF.neutron.endpoint_override
+        if neutron_endpoint is None:
+            endpoint_data = sess.get_endpoint_data(
                 service_type='network', interface=CONF.neutron.endpoint_type,
                 region_name=CONF.neutron.region_name)
             neutron_endpoint = endpoint_data.catalog_url
 
-        kwargs = {
-            'token': context.auth_token,
-            'endpoint_url': neutron_endpoint,
-            'insecure': CONF.neutron.insecure,
-            'ca_cert': CONF.neutron.ca_certificates_file
-        }
+        user_auth = token_endpoint.Token(neutron_endpoint, context.auth_token)
+        user_sess = session.Session(auth=user_auth)
 
-        return neutron_client.Client(NEUTRON_VERSION, **kwargs)
+        conn = openstack.connection.Connection(
+            session=user_sess, oslo_conf=CONF)
+        return conn.network
 
 
 class GlanceAuth(object):

octavia/common/config.py

@@ -37,10 +37,6 @@
 
 LOG = logging.getLogger(__name__)
 
-EXTRA_LOG_LEVEL_DEFAULTS = [
-    'neutronclient.v2_0.client=INFO',
-]
-
 core_opts = [
     cfg.HostnameOpt('host', default=utils.get_hostname(),
                     sample_default='<server-hostname.example.com>',
@@ -89,7 +85,7 @@
                        'octavia.api.drivers entrypoint.'),
                 default={'amphora': 'The Octavia Amphora driver.',
                          'octavia': 'Deprecated alias of the Octavia Amphora '
-                         'driver.',
+                                    'driver.',
                          }),
     cfg.StrOpt('default_provider_driver', default='amphora',
                help=_('Default provider driver.')),
@@ -747,21 +743,27 @@
 ]
 
 neutron_opts = [
-    cfg.StrOpt('service_name',
-               help=_('The name of the neutron service in the '
-                      'keystone catalog')),
     cfg.StrOpt('endpoint', help=_('A new endpoint to override the endpoint '
-                                  'in the keystone catalog.')),
-    cfg.StrOpt('region_name',
-               help=_('Region in Identity service catalog to use for '
-                      'communication with the OpenStack services.')),
-    cfg.StrOpt('endpoint_type', default='publicURL',
-               help=_('Endpoint interface in identity service to use')),
+                                  'in the keystone catalog.'),
+               deprecated_for_removal=True,
+               deprecated_reason=_('The endpoint_override option defined by '
+                                   'keystoneauth1 is the new name for this '
+                                   'option.'),
+               deprecated_since='2023.2/Bobcat'),
+    cfg.StrOpt('endpoint_type', help=_('Endpoint interface in identity '
+                                       'service to use'),
+               deprecated_for_removal=True,
+               deprecated_reason=_('This option was replaced by the '
+                                   'valid_interfaces option defined by '
+                                   'keystoneauth.'),
+               deprecated_since='2023.2/Bobcat'),
     cfg.StrOpt('ca_certificates_file',
-               help=_('CA certificates file path')),
-    cfg.BoolOpt('insecure',
-                default=False,
-                help=_('Disable certificate validation on SSL connections ')),
+               help=_('CA certificates file path'),
+               deprecated_for_removal=True,
+               deprecated_reason=_('The cafile option defined by '
+                                   'keystoneauth1 is the new name for this '
+                                   'option.'),
+               deprecated_since='2023.2/Bobcat'),
 ]
 
 glance_opts = [
@@ -902,15 +904,48 @@
 db_options.set_defaults(cfg.CONF, connection=_SQL_CONNECTION_DEFAULT,
                         max_pool_size=10, max_overflow=20, pool_timeout=10)
 
-ks_loading.register_auth_conf_options(cfg.CONF, constants.SERVICE_AUTH)
-ks_loading.register_session_conf_options(cfg.CONF, constants.SERVICE_AUTH)
+
+def register_ks_options(group):
+    ks_loading.register_auth_conf_options(cfg.CONF, group)
+    ks_loading.register_session_conf_options(cfg.CONF, group)
+    ks_loading.register_adapter_conf_options(cfg.CONF, group,
+                                             include_deprecated=False)
+
+
+register_ks_options(constants.SERVICE_AUTH)
+register_ks_options('neutron')
 
 
 def register_cli_opts():
     cfg.CONF.register_cli_opts(core_cli_opts)
     logging.register_options(cfg.CONF)
 
 
+def handle_neutron_deprecations():
+    # Apply neutron deprecated options to their new setting if needed
+
+    # Basicaly: if the value of the deprecated option is not the default:
+    # * convert it to a valid "new" value if needed
+    # * set it as the default for the new option
+    # Thus [neutron].<new_option> has an higher precedence than
+    # [neutron].<deprecated_option>
+    loc = cfg.CONF.get_location('endpoint', 'neutron')
+    if loc and loc.location != cfg.Locations.opt_default:
+        cfg.CONF.set_default('endpoint_override', cfg.CONF.neutron.endpoint,
+                             'neutron')
+
+    loc = cfg.CONF.get_location('endpoint_type', 'neutron')
+    if loc and loc.location != cfg.Locations.opt_default:
+        endpoint_type = cfg.CONF.neutron.endpoint_type.replace('URL', '')
+        cfg.CONF.set_default('valid_interfaces', [endpoint_type],
+                             'neutron')
+
+    loc = cfg.CONF.get_location('ca_certificates_file', 'neutron')
+    if loc and loc.location != cfg.Locations.opt_default:
+        cfg.CONF.set_default('cafile', cfg.CONF.neutron.ca_certificates_file,
+                             'neutron')
+
+
 def init(args, **kwargs):
     register_cli_opts()
     cfg.CONF(args=args, project='octavia',
@@ -920,14 +955,20 @@ def init(args, **kwargs):
     setup_remote_debugger()
     validate.check_default_ciphers_prohibit_list_conflict()
 
+    # Override default auth_type for plugins with the default from service_auth
+    auth_type = cfg.CONF.service_auth.auth_type
+    cfg.CONF.set_default('auth_type', auth_type, 'neutron')
+
+    handle_neutron_deprecations()
+
 
 def setup_logging(conf):
     """Sets up the logging options for a log with supplied name.
 
     :param conf: a cfg.ConfOpts object
     """
-    logging.set_defaults(default_log_levels=logging.get_default_log_levels() +
-                         EXTRA_LOG_LEVEL_DEFAULTS)
+    ll = logging.get_default_log_levels()
+    logging.set_defaults(default_log_levels=ll)
     product_name = "octavia"
     logging.setup(conf, product_name)
     LOG.info("Logging enabled!")

octavia/common/keystone.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from keystoneauth1 import exceptions as ks_exceptions
 from keystoneauth1 import loading as ks_loading
 from keystonemiddleware import auth_token
 from oslo_config import cfg
@@ -32,14 +33,17 @@ def __init__(self, section=constants.SERVICE_AUTH):
         self._auth = None
 
         self.section = section
-        ks_loading.register_auth_conf_options(cfg.CONF, self.section)
-        ks_loading.register_session_conf_options(cfg.CONF, self.section)
 
-    def get_session(self):
+    def get_session(self, auth=None):
         """Initializes a Keystone session.
 
         :return: a Keystone Session object
         """
+        if auth:
+            # Do not use the singleton with custom auth params
+            return ks_loading.load_session_from_conf_options(
+                cfg.CONF, self.section, auth=auth)
+
         if not self._session:
             self._session = ks_loading.load_session_from_conf_options(
                 cfg.CONF, self.section, auth=self.get_auth())
@@ -48,8 +52,57 @@ def get_session(self):
 
     def get_auth(self):
         if not self._auth:
-            self._auth = ks_loading.load_auth_from_conf_options(
-                cfg.CONF, self.section)
+            try:
+                self._auth = ks_loading.load_auth_from_conf_options(
+                    cfg.CONF, self.section)
+            except ks_exceptions.auth_plugins.MissingRequiredOptions as e:
+                if self.section == constants.SERVICE_AUTH:
+                    raise e
+                # NOTE(gthiemonge): MissingRequiredOptions is raised: there is
+                # one or more missing auth options in the config file. It may
+                # be due to the migration from python-neutronclient to
+                # openstacksdk.
+                # With neutronclient, most of the auth settings were in
+                # [service_auth] with a few overrides in [neutron],
+                # but with openstacksdk, we have all the auth settings in the
+                # [neutron] section. In order to support smooth upgrades, in
+                # case those options are missing, we override the undefined
+                # options with the existing settings from [service_auth].
+
+                # This code should be removed when all the deployment tools set
+                # the correct options in [neutron]
+
+                # The config options are lazily registered/loaded by keystone,
+                # it means that we cannot get/set them before invoking
+                # 'load_auth_from_conf_options' on 'service_auth'.
+                ks_loading.load_auth_from_conf_options(
+                    cfg.CONF, constants.SERVICE_AUTH)
+
+                config = getattr(cfg.CONF, self.section)
+                for opt in config:
+                    # For each option in the [neutron] section, get its setting
+                    # location, if the location is 'opt_default' or
+                    # 'set_default', it means that the option is not configured
+                    # in the config file, it should be replaced with the one
+                    # from [service_auth]
+                    loc = cfg.CONF.get_location(opt, self.section)
+                    if not loc or loc.location in (cfg.Locations.opt_default,
+                                                   cfg.Locations.set_default):
+                        if hasattr(cfg.CONF.service_auth, opt):
+                            cur_value = getattr(config, opt)
+                            value = getattr(cfg.CONF.service_auth, opt)
+                            if value != cur_value:
+                                log_value = (value if opt != "password"
+                                             else "<hidden>")
+                                LOG.debug("Overriding [%s].%s with '%s'",
+                                          self.section, opt, log_value)
+                                cfg.CONF.set_override(opt, value, self.section)
+
+                # Now we can call load_auth_from_conf_options for this specific
+                # service with the newly defined options.
+                self._auth = ks_loading.load_auth_from_conf_options(
+                    cfg.CONF, self.section)
+
         return self._auth
 
     def get_service_user_id(self):