Merge pull request StackStorm#2437 from lakshmi-kannan/gunicorn_confi…

…g_fixes

RFR: Gunicorn config fixes

CHANGELOG.rst

@@ -4,6 +4,13 @@ Changelog
 in development
 --------------
 
+* Changes to gunicorn configuration for both st2api and st2auth so common service
+  setup code is only run in workers and not master. (bug-fix)
+* Fix an issue where trigger watcher cannot get messages from queue if multiple API
+  processes are spun up. Now each trigger watcher gets its own queue and therefore
+  there are no locking issues. (bug-fix)
+* Dev environment by default now uses gunicorn to spin API and AUTH processes. (improvement)
+
 1.3.0 - January 22, 2016
 ------------------------
 

st2api/st2api/app.py

@@ -19,16 +19,17 @@
 from oslo_config import cfg
 from pecan.middleware.static import StaticFileMiddleware
 
+from st2api import config as st2api_config
 from st2common import hooks
 from st2common import log as logging
 from st2common.constants.system import VERSION_STRING
-
+from st2common.service_setup import setup as common_setup
 
 LOG = logging.getLogger(__name__)
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-def __get_pecan_config():
+def _get_pecan_config():
     opts = cfg.CONF.api_pecan
 
     cfg_dict = {
@@ -47,15 +48,33 @@ def __get_pecan_config():
 
 
 def setup_app(config=None):
-    LOG.info(VERSION_STRING)
-    LOG.info('Creating %s as Pecan app.' % __name__)
+    LOG.info('Creating st2api: %s as Pecan app.', VERSION_STRING)
+
+    is_gunicorn = getattr(config, 'is_gunicorn', False)
+    if is_gunicorn:
+        st2api_config.register_opts()
+        # This should be called in gunicorn case because we only want
+        # workers to connect to db, rabbbitmq etc. In standalone HTTP
+        # server case, this setup would have already occurred.
+        common_setup(service='api', config=st2api_config, setup_db=True,
+                     register_mq_exchanges=True,
+                     register_signal_handlers=True,
+                     register_internal_trigger_types=True,
+                     run_migrations=True,
+                     config_args=config.config_args)
 
     if not config:
-        config = __get_pecan_config()
+        # standalone HTTP server case
+        config = _get_pecan_config()
+    else:
+        # gunicorn case
+        if is_gunicorn:
+            config.app = _get_pecan_config().app
 
     app_conf = dict(config.app)
 
-    active_hooks = [hooks.RequestIDHook(), hooks.JSONErrorResponseHook(), hooks.LoggingHook()]
+    active_hooks = [hooks.RequestIDHook(), hooks.JSONErrorResponseHook(),
+                    hooks.LoggingHook()]
 
     if cfg.CONF.auth.enable:
         active_hooks.append(hooks.AuthHook())

st2api/st2api/cmd/api.py

@@ -27,6 +27,7 @@
 from st2api.signal_handlers import register_api_signal_handlers
 from st2api.listener import get_listener_if_set
 from st2api import config
+config.register_opts()
 from st2api import app
 
 __all__ = [

st2api/st2api/config.py

@@ -78,6 +78,3 @@ def _register_app_opts():
                    help='location of the logging.conf file')
     ]
     CONF.register_opts(logging_opts, group='api')
-
-
-register_opts()

st2api/st2api/controllers/v1/webhooks.py

@@ -51,11 +51,12 @@ def __init__(self, *args, **kwargs):
         self._trigger_types = WEBHOOK_TRIGGER_TYPES.keys()
 
         self._trigger_dispatcher = TriggerDispatcher(LOG)
+        queue_suffix = self.__class__.__name__
         self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger,
                                                update_handler=self._handle_update_trigger,
                                                delete_handler=self._handle_delete_trigger,
                                                trigger_types=self._trigger_types,
-                                               queue_suffix=self.__class__.__name__,
+                                               queue_suffix=queue_suffix,
                                                exclusive=True)
         self._trigger_watcher.start()
         self._register_webhook_trigger_types()

st2api/st2api/gunicorn_config.py

@@ -22,33 +22,15 @@
 
 import os
 
-from oslo_config import cfg
-
-from st2api import config  # noqa
-from st2common.service_setup import setup as common_setup
-
 __all__ = [
-    'server',
     'app'
 ]
 
-DEFAULT_ST2_CONFIG_PATH = '/etc/st2/st2.conf'
-ST2_CONFIG_PATH = os.environ.get('ST2_CONFIG_PATH', DEFAULT_ST2_CONFIG_PATH)
+bind = '127.0.0.1:9101'
 
-CONFIG_ARGS = ['--config-file', ST2_CONFIG_PATH]
-common_setup(service='api', config=config, setup_db=True, register_mq_exchanges=True,
-             register_signal_handlers=False, register_internal_trigger_types=True,
-             config_args=CONFIG_ARGS)
-
-server = {
-    'host': cfg.CONF.api.host,
-    'port': cfg.CONF.api.port
-}
+config_args = ['--config-file', os.environ.get('ST2_CONFIG_PATH', '/etc/st2/st2.conf')]
+is_gunicorn = True
 
 app = {
-    'root': 'st2api.controllers.root.RootController',
-    'modules': ['st2api'],
-    'debug': cfg.CONF.api_pecan.debug,
-    'errors': {'__force_dict__': True},
-    'guess_content_type_from_ext': False
+    'modules': ['st2api']
 }

st2auth/st2auth/app.py

@@ -16,9 +16,11 @@
 import pecan
 from oslo_config import cfg
 
+from st2auth import config as st2auth_config
 from st2common import hooks
 from st2common import log as logging
-
+from st2common.constants.system import VERSION_STRING
+from st2common.service_setup import setup as common_setup
 
 LOG = logging.getLogger(__name__)
 
@@ -38,15 +40,37 @@ def _get_pecan_config():
 
 
 def setup_app(config=None):
+    LOG.info('Creating st2auth: %s as Pecan app.', VERSION_STRING)
+
+    is_gunicorn = getattr(config, 'is_gunicorn', False)
+    if is_gunicorn:
+        # This should be called in gunicorn case because we only want
+        # workers to connect to db, rabbbitmq etc. In standalone HTTP
+        # server case, this setup would have already occurred.
+        st2auth_config.register_opts()
+        common_setup(service='auth', config=st2auth_config, setup_db=True,
+                     register_mq_exchanges=False,
+                     register_signal_handlers=True,
+                     register_internal_trigger_types=False,
+                     run_migrations=False,
+                     config_args=config.config_args)
 
     if not config:
+        # standalone HTTP server case
         config = _get_pecan_config()
+    else:
+        # gunicorn case
+        if is_gunicorn:
+            config.app = _get_pecan_config().app
 
     app_conf = dict(config.app)
 
-    return pecan.make_app(
+    app = pecan.make_app(
         app_conf.pop('root'),
         logging=getattr(config, 'logging', {}),
         hooks=[hooks.JSONErrorResponseHook(), hooks.CorsHook()],
         **app_conf
     )
+    LOG.info('%s app created.' % __name__)
+
+    return app

st2auth/st2auth/cmd/api.py

@@ -25,6 +25,7 @@
 from st2common.service_setup import teardown as common_teardown
 from st2common.constants.auth import VALID_MODES
 from st2auth import config
+config.register_opts()
 from st2auth import app
 
 

st2auth/st2auth/config.py

@@ -71,5 +71,3 @@ def _register_app_opts():
                     help='List of origins allowed'),
     ]
     cfg.CONF.register_cli_opts(api_opts, group='api')
-
-register_opts()

st2auth/st2auth/gunicorn_config.py

@@ -22,32 +22,15 @@
 
 import os
 
-from oslo_config import cfg
-
-from st2auth import config  # noqa
-from st2common.service_setup import setup as common_setup
-
 __all__ = [
-    'server',
     'app'
 ]
 
-DEFAULT_ST2_CONFIG_PATH = '/etc/st2/st2.conf'
-ST2_CONFIG_PATH = os.environ.get('ST2_CONFIG_PATH', DEFAULT_ST2_CONFIG_PATH)
+bind = '127.0.0.1:9100'
 
-CONFIG_ARGS = ['--config-file', ST2_CONFIG_PATH]
-common_setup(service='api', config=config, setup_db=True, register_mq_exchanges=True,
-             register_signal_handlers=False, register_internal_trigger_types=True,
-             config_args=CONFIG_ARGS)
-
-server = {
-    'host': cfg.CONF.auth.host,
-    'port': cfg.CONF.auth.port
-}
+config_args = ['--config-file', os.environ.get('ST2_CONFIG_PATH', '/etc/st2/st2.conf')]
+is_gunicorn = True
 
 app = {
-    'root': 'st2auth.controllers.root.RootController',
-    'modules': ['st2auth'],
-    'debug': cfg.CONF.auth.debug,
-    'errors': {'__force_dict__': True}
+    'modules': ['st2auth']
 }

st2common/st2common/log.py

@@ -23,7 +23,6 @@
 from functools import wraps
 
 import six
-from oslo_config import cfg
 
 from st2common.logging.filters import ExclusionFilter
 
@@ -118,18 +117,18 @@ def _audit(logger, msg, *args, **kwargs):
 logging.Logger.audit = _audit
 
 
-def _add_exclusion_filters(handlers):
-    for h in handlers:
-        h.addFilter(ExclusionFilter(cfg.CONF.log.excludes))
+def _add_exclusion_filters(handlers, excludes=None):
+    if excludes:
+        for h in handlers:
+            h.addFilter(ExclusionFilter(excludes))
 
 
 def _redirect_stderr():
     # It is ok to redirect stderr as none of the st2 handlers write to stderr.
-    if cfg.CONF.log.redirect_stderr:
-        sys.stderr = LoggingStream('STDERR')
+    sys.stderr = LoggingStream('STDERR')
 
 
-def setup(config_file, disable_existing_loggers=False):
+def setup(config_file, redirect_stderr=True, excludes=None, disable_existing_loggers=False):
     """
     Configure logging from file.
     """
@@ -139,7 +138,8 @@ def setup(config_file, disable_existing_loggers=False):
                                   disable_existing_loggers=disable_existing_loggers)
         handlers = logging.getLoggerClass().manager.root.handlers
         _add_exclusion_filters(handlers)
-        _redirect_stderr()
+        if redirect_stderr:
+            _redirect_stderr()
     except Exception as exc:
         # revert stderr redirection since there is no logger in place.
         sys.stderr = sys.__stderr__

st2common/st2common/service_setup.py

@@ -66,7 +66,7 @@ def setup(service, config, setup_db=True, register_mq_exchanges=True,
     """
     # Set up logger which logs everything which happens during and before config
     # parsing to sys.stdout
-    logging.setup(DEFAULT_LOGGING_CONF_PATH)
+    logging.setup(DEFAULT_LOGGING_CONF_PATH, excludes=None)
 
     # Parse args to setup config.
     if config_args:
@@ -83,7 +83,8 @@ def setup(service, config, setup_db=True, register_mq_exchanges=True,
     logging_config_path = os.path.abspath(logging_config_path)
 
     LOG.debug('Using logging config: %s', logging_config_path)
-    logging.setup(logging_config_path)
+    logging.setup(logging_config_path, redirect_stderr=cfg.CONF.log.redirect_stderr,
+                  excludes=cfg.CONF.log.excludes)
 
     if cfg.CONF.debug:
         enable_debugging()

st2common/st2common/services/sensor_watcher.py

@@ -17,13 +17,13 @@
 # XXX: Refactor.
 
 import eventlet
-import uuid
 from kombu.mixins import ConsumerMixin
 from kombu import Connection
 
 from st2common import log as logging
 from st2common.transport import reactor, publishers
 from st2common.transport import utils as transport_utils
+import st2common.util.queues as queue_utils
 
 LOG = logging.getLogger(__name__)
 
@@ -103,9 +103,8 @@ def stop(self):
 
     @staticmethod
     def _get_queue(queue_suffix):
-        if not queue_suffix:
-            # pick last 10 digits of uuid. Arbitrary but unique enough for the TriggerWatcher.
-            u_hex = uuid.uuid4().hex
-            queue_suffix = uuid.uuid4().hex[len(u_hex) - 10:]
-        queue_name = 'st2.sensor.watch.%s' % queue_suffix
+        queue_name = queue_utils.get_queue_name(queue_name_base='st2.sensor.watch',
+                                                queue_name_suffix=queue_suffix,
+                                                add_random_uuid_to_suffix=True
+                                                )
         return reactor.get_sensor_cud_queue(queue_name, routing_key='#')

st2common/st2common/services/triggerwatcher.py

@@ -14,14 +14,14 @@
 # limitations under the License.
 
 import eventlet
-import uuid
 from kombu.mixins import ConsumerMixin
 from kombu import Connection
 
 from st2common import log as logging
 from st2common.persistence.trigger import Trigger
 from st2common.transport import reactor, publishers
 from st2common.transport import utils as transport_utils
+import st2common.util.queues as queue_utils
 
 LOG = logging.getLogger(__name__)
 
@@ -141,9 +141,8 @@ def _load_triggers_from_db(self):
 
     @staticmethod
     def _get_queue(queue_suffix, exclusive):
-        if not queue_suffix:
-            # pick last 10 digits of uuid. Arbitrary but unique enough for the TriggerWatcher.
-            u_hex = uuid.uuid4().hex
-            queue_suffix = uuid.uuid4().hex[len(u_hex) - 10:]
-        queue_name = 'st2.trigger.watch.%s' % queue_suffix
+        queue_name = queue_utils.get_queue_name(queue_name_base='st2.trigger.watch',
+                                                queue_name_suffix=queue_suffix,
+                                                add_random_uuid_to_suffix=True
+                                                )
         return reactor.get_trigger_cud_queue(queue_name, routing_key='#', exclusive=exclusive)

st2common/st2common/triggers.py

@@ -47,9 +47,8 @@ def _register_internal_trigger_type(trigger_definition):
                           trigger_type_db.get_reference().ref)
             raise
         except StackStormDBObjectConflictError:
-            LOG.exception('Shadow trigger creation of "%s" failed with uniqueness conflict.',
-                          trigger_type_db.get_reference().ref)
-            raise
+            LOG.debug('Shadow trigger "%s" already exists. Ignoring.',
+                      trigger_type_db.get_reference().ref, exc_info=True)
 
     return trigger_type_db
 
@@ -75,8 +74,8 @@ def register_internal_trigger_types():
                 trigger_type_db = _register_internal_trigger_type(
                     trigger_definition=trigger_definition)
             except:
-                LOG.warning('Failed registering internal trigger: %s.', trigger_definition,
-                            exc_info=True)
+                LOG.exception('Failed registering internal trigger: %s.', trigger_definition)
+                raise
             else:
                 registered_trigger_types_db.append(trigger_type_db)