Hello All,

In continuation to below, further I am facing new issues w.r.t python versions compatibility.  Requests for some inputs here on how can I solve this ?

/var/lib/openstack/lib64/python3.6/site-packages/scss/namespace.py:172: DeprecationWarning: inspect.getargspec() is deprecated since Python 3.0, use inspect.signature() or inspect.getfullargspec()

Complete pod logs :

+ COMMAND=start

+ start

++ python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'

+ SITE_PACKAGES_ROOT=/var/lib/openstack/lib/python3.6/site-packages

+ rm -f /var/lib/openstack/lib/python3.6/site-packages/openstack_dashboard/local/local_settings.py

+ ln -s /etc/openstack-dashboard/local_settings /var/lib/openstack/lib/python3.6/site-packages/openstack_dashboard/local/local_settings.py

+ chown -R horizon /var/lib/openstack/lib/python3.6/site-packages/openstack_dashboard/local/

+ a2enmod headers

+ a2enmod rewrite

+ a2dismod status

/tmp/horizon.sh: line 17: :wq: command not found

+ sed -i 's/LoadModule status_module/#LoadModule status_module/' /etc/httpd/conf.modules.d/00-base.conf

+ '[' -f /etc/apache2/envvars ']'

+ rm -rf /var/run/apache2/htcacheclean

+ APACHE_DIR=apache2

+ PANEL_DIR=/var/lib/openstack/lib/python3.6/site-packages/heat_dashboard/enabled

+ '[' -d /var/lib/openstack/lib/python3.6/site-packages/heat_dashboard/enabled ']'

+ unset PANEL_DIR

+ PANEL_DIR=/var/lib/openstack/lib/python3.6/site-packages/neutron_taas_dashboard/enabled

+ '[' -d /var/lib/openstack/lib/python3.6/site-packages/neutron_taas_dashboard/enabled ']'

+ unset PANEL_DIR

+ type -p gettext

+ cd /var/lib/openstack/lib/python3.6/site-packages/openstack_dashboard

+ /tmp/manage.py compilemessages

/var/lib/openstack/lib64/python3.6/site-packages/scss/compiler.py:1430: DeprecationWarning: invalid escape sequence \:

  result = tb * (i + nesting) + "@media -sass-debug-info{filename{font-family:file\:\/\/%s}line{font-family:\\00003%s}}" % (filename, lineno) + nl

/var/lib/openstack/lib64/python3.6/site-packages/scss/cssdefs.py:516: DeprecationWarning: invalid escape sequence \s

  ''', re.VERBOSE)

/var/lib/openstack/lib64/python3.6/site-packages/scss/namespace.py:172: DeprecationWarning: inspect.getargspec() is deprecated since Python 3.0, use inspect.signature() or inspect.getfullargspec()

  argspec = inspect.getargspec(function)

Traceback (most recent call last):

File "/tmp/manage.py", line 19, in <module>

    execute_from_command_line(sys.argv)

  File "/var/lib/openstack/lib/python3.6/site-packages/django/core/management/__init__.py", line 381, in execute_from_command_line

    utility.execute()

  File "/var/lib/openstack/lib/python3.6/site-packages/django/core/management/__init__.py", line 357, in execute

    django.setup()

  File "/var/lib/openstack/lib/python3.6/site-packages/django/__init__.py", line 24, in setup

    apps.populate(settings.INSTALLED_APPS)

  File "/var/lib/openstack/lib/python3.6/site-packages/django/apps/registry.py", line 114, in populate

    app_config.import_models()

  File "/var/lib/openstack/lib/python3.6/site-packages/django/apps/config.py", line 211, in import_models

    self.models_module = import_module(models_module_name)

  File "/usr/lib64/python3.6/importlib/__init__.py", line 126, in import_module

    return _bootstrap._gcd_import(name[level:], package, level)

  File "<frozen importlib._bootstrap>", line 994, in _gcd_import

  File "<frozen importlib._bootstrap>", line 971, in _find_and_load

  File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked

  File "<frozen importlib._bootstrap>", line 665, in _load_unlocked

  File "<frozen importlib._bootstrap_external>", line 678, in exec_module

  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed

  File "/var/lib/openstack/lib/python3.6/site-packages/django/contrib/auth/models.py", line 2, in <module>

    from django.contrib.auth.base_user import AbstractBaseUser, BaseUserManager

  File "/var/lib/openstack/lib/python3.6/site-packages/django/contrib/auth/base_user.py", line 47, in <module>

    class AbstractBaseUser(models.Model):

  File "/var/lib/openstack/lib/python3.6/site-packages/django/db/models/base.py", line 117, in __new__

    new_class.add_to_class('_meta', Options(meta, app_label))

  File "/var/lib/openstack/lib/python3.6/site-packages/django/db/models/base.py", line 321, in add_to_class

    value.contribute_to_class(cls, name)

  File "/var/lib/openstack/lib/python3.6/site-packages/django/db/models/options.py", line 204, in contribute_to_class

    self.db_table = truncate_name(self.db_table, connection.ops.max_name_length())

  File "/var/lib/openstack/lib/python3.6/site-packages/django/db/__init__.py", line 28, in __getattr__

    return getattr(connections[DEFAULT_DB_ALIAS], item)

  File "/var/lib/openstack/lib/python3.6/site-packages/django/db/utils.py", line 201, in __getitem__

    backend = load_backend(db['ENGINE'])

  File "/var/lib/openstack/lib/python3.6/site-packages/django/db/utils.py", line 110, in load_backend

    return import_module('%s.base' % backend_name)

  File "/usr/lib64/python3.6/importlib/__init__.py", line 126, in import_module

    return _bootstrap._gcd_import(name[level:], package, level)

  File "/var/lib/openstack/lib/python3.6/site-packages/django/db/backends/mysql/base.py", line 36, in <module>

    raise ImproperlyConfigured('mysqlclient 1.3.13 or newer is required; you have %s.' % Database.__version__)

django.core.exceptions.ImproperlyConfigured: mysqlclient 1.3.13 or newer is required; you have 0.9.3.

Regards,

Haridhar Kalvala

From: Hu, Yong <yong.hu@intel.com>
Sent: Monday, January 11, 2021 1:58 PM
To: Kalvala, Haridhar <haridhar.kalvala@intel.com>; starlingx-discuss@lists.starlingx.io
Cc: Bailey, Henry Albert (Al) <Al.Bailey@windriver.com>; Penney, Don <Don.Penney@windriver.com>; Miller, Frank <Frank.Miller@windriver.com>; Mukherjee, Sanjay K <sanjay.k.mukherjee@intel.com>; Bhat, Gopalkrishna <gopalkrishna.bhat@intel.com>
Subject: Re: [Starlingx-discuss] Horizon container deployment, Pod status (CrashLoopBackOff) issue.

You might get some hints by inspecting the pod by:

# kubectl -n openstack describe pod horizon-d777d8b7d-cpvq7                             

# kubectl -n openstack logs horizon-d777d8b7d-cpvq7                             

From: "Kalvala, Haridhar" <haridhar.kalvala@intel.com>
Date: Monday, January 11, 2021 at 4:19 PM
To: "starlingx-discuss@lists.starlingx.io" <starlingx-discuss@lists.starlingx.io>
Cc: "Bailey, Henry Albert (Al)" <Al.Bailey@windriver.com>, "Penney, Don" <Don.Penney@windriver.com>, "Miller, Frank" <Frank.Miller@windriver.com>, "Mukherjee, Sanjay K" <sanjay.k.mukherjee@intel.com>, "Bhat, Gopalkrishna" <gopalkrishna.bhat@intel.com>
Subject: [Starlingx-discuss] Horizon container deployment, Pod status (CrashLoopBackOff) issue.

Hello all,

Any pointers on how can I debug below CrashLoopBackOff  status issue for  horizon container image.

Pod status :

horizon-d777d8b7d-cpvq7                              0/1     CrashLoopBackOff   43         3h17m

horizon-db-init-4pcd4                                0/1     Completed          0          3h17m

horizon-db-sync-26ltd                                0/1     Completed          0          3h17m

Error Log:

2021-01-11 16:06:21.812 414 DEBUG armada.handlers.wait [-] [chart=openstack-horizon]: pod horizon-d777d8b7d-cpvq7 not ready: Waiting for pod horizon-d777d8b7d-cpvq7 to be ready... handle_resource /usr/local/lib/python3.6/dist-packages/armada/handlers/wait.py:260[00m

2021-01-11 16:07:17.488 414 DEBUG armada.handlers.lock [-] Updating lock update_lock /usr/local/lib/python3.6/dist-packages/armada/handlers/lock.py:176[00m

2021-01-11 16:08:17.612 414 DEBUG armada.handlers.lock [-] Updating lock update_lock /usr/local/lib/python3.6/dist-packages/armada/handlers/lock.py:176[00m

2021-01-11 16:08:19.221 414 ERROR armada.handlers.wait [-] [chart=openstack-horizon]: Timed out waiting for pods (namespace=openstack, labels=(release_group=osh-openstack-horizon)). These pods were not ready=['horizon-d777d8b7d-cpvq7'][00m

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada [-] Chart deploy [openstack-horizon] failed: armada.exceptions.k8s_exceptions.KubernetesWatchTimeoutException: Timed out waiting for pods (namespace=openstack, labels=(release_group=osh-openstack-horizon)). These pods were not ready=['horizon-d777d8b7d-cpvq7']

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada Traceback (most recent call last):

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/local/lib/python3.6/dist-packages/armada/handlers/armada.py", line 225, in handle_result

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     result = get_result()

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/lib/python3.6/concurrent/futures/_base.py", line 425, in result

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     return self.__get_result()

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/lib/python3.6/concurrent/futures/_base.py", line 384, in __get_result

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     raise self._exception

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/lib/python3.6/concurrent/futures/thread.py", line 56, in run

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     result = self.fn(*self.args, **self.kwargs)

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/local/lib/python3.6/dist-packages/armada/handlers/armada.py", line 214, in deploy_chart

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     chart, cg_test_all_charts, prefix, known_releases)

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/local/lib/python3.6/dist-packages/armada/handlers/chart_deploy.py", line 248, in execute

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     chart_wait.wait(timer)

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/local/lib/python3.6/dist-packages/armada/handlers/wait.py", line 134, in wait

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     wait.wait(timeout=timeout)

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/local/lib/python3.6/dist-packages/armada/handlers/wait.py", line 294, in wait

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     modified = self._wait(deadline)

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada   File "/usr/local/lib/python3.6/dist-packages/armada/handlers/wait.py", line 354, in _wait

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada     raise k8s_exceptions.KubernetesWatchTimeoutException(error)

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada armada.exceptions.k8s_exceptions.KubernetesWatchTimeoutException: Timed out waiting for pods (namespace=openstack, labels=(release_group=osh-openstack-horizon)). These pods were not ready=['horizon-d777d8b7d-cpvq7']

2021-01-11 16:08:19.222 414 ERROR armada.handlers.armada [00m

2021-01-11 16:08:19.223 414 ERROR armada.handlers.armada [-] Chart deploy(s) failed: ['openstack-horizon'][00m

2021-01-11 16:08:19.625 414 INFO armada.handlers.lock [-] Releasing lock[00m

2021-01-11 16:08:19.668 414 ERROR armada.cli [-] Caught internal exception: armada.exceptions.armada_exceptions.ChartDeployException: Exception deploying charts: ['openstack-horizon']

2021-01-11 16:08:19.668 414 ERROR armada.cli Traceback (most recent call last):

2021-01-11 16:08:19.668 414 ERROR armada.cli   File "/usr/local/lib/python3.6/dist-packages/armada/cli/__init__.py", line 38, in safe_invoke

2021-01-11 16:08:19.668 414 ERROR armada.cli     self.invoke()

2021-01-11 16:08:19.668 414 ERROR armada.cli   File "/usr/local/lib/python3.6/dist-packages/armada/cli/apply.py", line 213, in invoke

2021-01-11 16:08:19.668 414 ERROR armada.cli     resp = self.handle(documents, tiller)

2021-01-11 16:08:19.668 414 ERROR armada.cli   File "/usr/local/lib/python3.6/dist-packages/armada/handlers/lock.py", line 81, in func_wrapper

2021-01-11 16:08:19.668 414 ERROR armada.cli     return future.result()

2021-01-11 16:08:19.668 414 ERROR armada.cli   File "/usr/lib/python3.6/concurrent/futures/_base.py", line 425, in result

2021-01-11 16:08:19.668 414 ERROR armada.cli     return self.__get_result()

2021-01-11 16:08:19.668 414 ERROR armada.cli   File "/usr/lib/python3.6/concurrent/futures/_base.py", line 384, in __get_result

2021-01-11 16:08:19.668 414 ERROR armada.cli     raise self._exception

2021-01-11 16:08:19.668 414 ERROR armada.cli   File "/usr/lib/python3.6/concurrent/futures/thread.py", line 56, in run

2021-01-11 16:08:19.668 414 ERROR armada.cli     result = self.fn(*self.args, **self.kwargs)

2021-01-11 16:08:19.668 414 ERROR armada.cli   File "/usr/local/lib/python3.6/dist-packages/armada/cli/apply.py", line 256, in handle

2021-01-11 16:08:19.668 414 ERROR armada.cli     return armada.sync()

2021-01-11 16:08:19.668 414 ERROR armada.cli   File "/usr/local/lib/python3.6/dist-packages/armada/handlers/armada.py", line 252, in sync

2021-01-11 16:08:19.668 414 ERROR armada.cli     raise armada_exceptions.ChartDeployException(failures)

2021-01-11 16:08:19.668 414 ERROR armada.cli armada.exceptions.armada_exceptions.ChartDeployException: Exception deploying charts: ['openstack-horizon']

2021-01-11 16:08:19.668 414 ERROR armada.cli [00m

command terminated with exit code 1

Regards,

Haridhar Kalvala