Merge pull request StackStorm#2669 from StackStorm/issue_167/user_sco…

…ped_vars

Introduce 'scope' field in key value model

.gitignore

@@ -39,6 +39,9 @@ nosetests.xml
 .idea
 .DS_Store
 ._*
+.vscode
+*.sublime-project
+*.sublime-workspace
 
 # Editor Saves
 *~

CHANGELOG.rst

@@ -30,6 +30,7 @@ in development
 * Introduce a new concept of pack config schemas. Each pack can now contain a
   ``config.schema.yaml`` file. This file can contain an optional schema for the pack config. In the
   future, pack configs will be validated against the schema (if available). (new feature)
+* Add data model and API changes for supporting user scoped variables. (new-feature, experimental)
 
 1.4.0 - April 18, 2016
 ----------------------

st2actions/st2actions/notifier/notifier.py

@@ -39,7 +39,7 @@
 from st2common.constants.action import ACTION_CONTEXT_KV_PREFIX
 from st2common.constants.action import ACTION_PARAMETERS_KV_PREFIX
 from st2common.constants.action import ACTION_RESULTS_KV_PREFIX
-from st2common.constants.system import SYSTEM_KV_PREFIX
+from st2common.constants.keyvalue import SYSTEM_SCOPE
 from st2common.services.keyvalues import KeyValueLookup
 
 __all__ = [
@@ -190,7 +190,7 @@ def _post_notify_subsection_triggers(self, liveaction=None, execution=None,
                 raise Exception('Failed notifications to routes: %s' % ', '.join(failed_routes))
 
     def _build_jinja_context(self, liveaction, execution):
-        context = {SYSTEM_KV_PREFIX: KeyValueLookup()}
+        context = {SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}
         context.update({ACTION_PARAMETERS_KV_PREFIX: liveaction.parameters})
         context.update({ACTION_CONTEXT_KV_PREFIX: liveaction.context})
         context.update({ACTION_RESULTS_KV_PREFIX: execution.result})

st2actions/st2actions/runners/actionchainrunner.py

@@ -29,7 +29,7 @@
 from st2common.constants.action import LIVEACTION_STATUS_CANCELED
 from st2common.constants.action import LIVEACTION_COMPLETED_STATES
 from st2common.constants.action import LIVEACTION_FAILED_STATES
-from st2common.constants.system import SYSTEM_KV_PREFIX
+from st2common.constants.keyvalue import SYSTEM_SCOPE
 from st2common.content.loader import MetaLoader
 from st2common.exceptions.action import (ParameterRenderingFailedException,
                                          InvalidActionReferencedException)
@@ -192,7 +192,7 @@ def _is_valid_node_name(self, all_node_names, node_name):
     def _get_rendered_vars(vars, action_parameters):
         if not vars:
             return {}
-        context = {SYSTEM_KV_PREFIX: KeyValueLookup()}
+        context = {SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}
         context.update(action_parameters)
         return jinja_utils.render_values(mapping=vars, context=context)
 
@@ -463,7 +463,7 @@ def _render_publish_vars(action_node, action_parameters, execution_result,
         context.update(previous_execution_results)
         context.update(chain_vars)
         context.update({RESULTS_KEY: previous_execution_results})
-        context.update({SYSTEM_KV_PREFIX: KeyValueLookup()})
+        context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)})
 
         try:
             rendered_result = jinja_utils.render_values(mapping=action_node.publish,
@@ -485,7 +485,7 @@ def _resolve_params(action_node, original_parameters, results, chain_vars, chain
         context.update(results)
         context.update(chain_vars)
         context.update({RESULTS_KEY: results})
-        context.update({SYSTEM_KV_PREFIX: KeyValueLookup()})
+        context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)})
         context.update({ACTION_CONTEXT_KV_PREFIX: chain_context})
         try:
             rendered_params = jinja_utils.render_values(mapping=action_node.get_parameters(),

st2api/st2api/controllers/resource.py

@@ -272,6 +272,26 @@ def _get_from_model_kwargs_for_request(self, request):
         """
         return self.from_model_kwargs
 
+    def _get_one_by_scope_and_name(self, scope, name, from_model_kwargs=None):
+        """
+        Retrieve an item given scope and name. Only KeyValuePair now has concept of 'scope'.
+
+        :param scope: Scope the key belongs to.
+        :type scope: ``str``
+
+        :param name: Name of the key.
+        :type name: ``str``
+        """
+        instance = self.access.get_by_scope_and_name(scope=scope, name=name)
+        if not instance:
+            msg = 'KeyValuePair with name: %s and scope: %s not found in db.' % (name, scope)
+            raise StackStormDBObjectNotFoundError(msg)
+        from_model_kwargs = from_model_kwargs or {}
+        result = self.model.from_model(instance, **from_model_kwargs)
+        LOG.debug('GET with scope=%s and name=%s, client_result=%s', scope, name, result)
+
+        return result
+
 
 class ContentPackResourceController(ResourceController):
     include_reference = False

st2api/st2api/controllers/root.py

@@ -18,8 +18,8 @@
 from st2common import __version__
 from st2common import log as logging
 from st2common.controllers import BaseRootController
-import st2api.controllers.v1.root as v1_root
 import st2api.controllers.exp.root as exp_root
+import st2api.controllers.v1.root as v1_root
 
 __all__ = [
     'RootController'
@@ -34,6 +34,7 @@ class RootController(BaseRootController):
     def __init__(self):
         v1_controller = v1_root.RootController()
         exp_controller = exp_root.RootController()
+
         self.controllers = {
             'v1': v1_controller,
             'exp': exp_controller

st2api/st2api/controllers/v1/keyvalue.py

@@ -17,13 +17,17 @@
 import six
 from mongoengine import ValidationError
 
+from st2api.controllers.resource import ResourceController
 from st2common import log as logging
-from st2common.exceptions.keyvalue import CryptoKeyNotSetupException
+from st2common.constants.keyvalue import SYSTEM_SCOPE, USER_SCOPE, ALLOWED_SCOPES
+from st2common.exceptions.db import StackStormDBObjectNotFoundError
+from st2common.exceptions.keyvalue import CryptoKeyNotSetupException, InvalidScopeException
 from st2common.models.api.keyvalue import KeyValuePairAPI
 from st2common.models.api.base import jsexpose
 from st2common.persistence.keyvalue import KeyValuePair
 from st2common.services import coordination
-from st2api.controllers.resource import ResourceController
+from st2common.services.keyvalues import get_key_reference
+from st2common.util.api import get_requester
 
 http_client = six.moves.http_client
 
@@ -42,29 +46,40 @@ class KeyValuePairController(ResourceController):
     model = KeyValuePairAPI
     access = KeyValuePair
     supported_filters = {
-        'prefix': 'name__startswith'
+        'prefix': 'name__startswith',
+        'scope': 'scope'
     }
 
     def __init__(self):
         super(KeyValuePairController, self).__init__()
         self._coordinator = coordination.get_coordinator()
         self.get_one_db_method = self._get_by_name
 
-    @jsexpose(arg_types=[str, bool])
-    def get_one(self, name, decrypt=False):
+    @jsexpose(arg_types=[str, str, bool])
+    def get_one(self, name, scope=SYSTEM_SCOPE, decrypt=False):
         """
             List key by name.
 
             Handle:
                 GET /keys/key1
         """
+        self._validate_scope(scope=scope)
+        key_ref = get_key_reference(scope=scope, name=name, user=get_requester())
         from_model_kwargs = {'mask_secrets': not decrypt}
-        kvp_db = super(KeyValuePairController, self)._get_one_by_name_or_id(name_or_id=name,
-            from_model_kwargs=from_model_kwargs)
-        return kvp_db
+        try:
+            kvp_api = self._get_one_by_scope_and_name(
+                name=key_ref,
+                scope=scope,
+                from_model_kwargs=from_model_kwargs
+            )
+        except StackStormDBObjectNotFoundError as e:
+            abort(http_client.NOT_FOUND, e.message)
+            return
+
+        return kvp_api
 
-    @jsexpose(arg_types=[str, bool])
-    def get_all(self, prefix=None, decrypt=False, **kwargs):
+    @jsexpose(arg_types=[str, str, bool])
+    def get_all(self, prefix=None, scope=SYSTEM_SCOPE, decrypt=False, **kwargs):
         """
             List all keys.
 
@@ -73,30 +88,48 @@ def get_all(self, prefix=None, decrypt=False, **kwargs):
         """
         from_model_kwargs = {'mask_secrets': not decrypt}
         kwargs['prefix'] = prefix
-        kvp_dbs = super(KeyValuePairController, self)._get_all(from_model_kwargs=from_model_kwargs,
-                                                               **kwargs)
-        return kvp_dbs
+        if scope:
+            self._validate_scope(scope=scope)
+            kwargs['scope'] = scope
 
-    @jsexpose(arg_types=[str, str], body_cls=KeyValuePairAPI)
-    def put(self, name, kvp):
+        if scope == USER_SCOPE and kwargs['prefix']:
+            kwargs['prefix'] = get_key_reference(name=kwargs['prefix'], scope=scope,
+                                                 user=get_requester())
+
+        kvp_apis = super(KeyValuePairController, self)._get_all(from_model_kwargs=from_model_kwargs,
+                                                                **kwargs)
+        return kvp_apis
+
+    @jsexpose(arg_types=[str, str, str], body_cls=KeyValuePairAPI)
+    def put(self, name, kvp, scope=SYSTEM_SCOPE):
         """
         Create a new entry or update an existing one.
         """
-        lock_name = self._get_lock_name_for_key(name=name)
-
+        scope = getattr(kvp, 'scope', scope)
+        self._validate_scope(scope=scope)
+        key_ref = get_key_reference(scope=scope, name=name, user=get_requester())
+        lock_name = self._get_lock_name_for_key(name=key_ref, scope=scope)
+        LOG.debug('PUT scope: %s, name: %s', scope, name)
         # TODO: Custom permission check since the key doesn't need to exist here
 
         # Note: We use lock to avoid a race
         with self._coordinator.get_lock(lock_name):
-            existing_kvp = self._get_by_name(resource_name=name)
+            try:
+                existing_kvp_api = self._get_one_by_scope_and_name(
+                    scope=scope,
+                    name=key_ref
+                )
+            except StackStormDBObjectNotFoundError:
+                existing_kvp_api = None
 
-            kvp.name = name
+            kvp.name = key_ref
+            kvp.scope = scope
 
             try:
                 kvp_db = KeyValuePairAPI.to_model(kvp)
 
-                if existing_kvp:
-                    kvp_db.id = existing_kvp.id
+                if existing_kvp_api:
+                    kvp_db.id = existing_kvp_api.id
 
                 kvp_db = KeyValuePair.add_or_update(kvp_db)
             except (ValidationError, ValueError) as e:
@@ -107,31 +140,45 @@ def put(self, name, kvp):
                 LOG.exception(str(e))
                 abort(http_client.BAD_REQUEST, str(e))
                 return
+            except InvalidScopeException as e:
+                LOG.exception(str(e))
+                abort(http_client.BAD_REQUEST, str(e))
+                return
         extra = {'kvp_db': kvp_db}
         LOG.audit('KeyValuePair updated. KeyValuePair.id=%s' % (kvp_db.id), extra=extra)
 
         kvp_api = KeyValuePairAPI.from_model(kvp_db)
         return kvp_api
 
-    @jsexpose(arg_types=[str], status_code=http_client.NO_CONTENT)
-    def delete(self, name):
+    @jsexpose(arg_types=[str, str], status_code=http_client.NO_CONTENT)
+    def delete(self, name, scope=SYSTEM_SCOPE):
         """
             Delete the key value pair.
 
             Handles requests:
                 DELETE /keys/1
         """
-        lock_name = self._get_lock_name_for_key(name=name)
+        self._validate_scope(scope=scope)
+        key_ref = get_key_reference(scope=scope, name=name, user=get_requester())
+        lock_name = self._get_lock_name_for_key(name=key_ref, scope=scope)
 
         # Note: We use lock to avoid a race
         with self._coordinator.get_lock(lock_name):
-            kvp_db = self._get_by_name(resource_name=name)
-
-            if not kvp_db:
-                abort(http_client.NOT_FOUND)
+            from_model_kwargs = {'mask_secrets': True}
+            try:
+                kvp_api = self._get_one_by_scope_and_name(
+                    name=key_ref,
+                    scope=scope,
+                    from_model_kwargs=from_model_kwargs
+                )
+            except StackStormDBObjectNotFoundError as e:
+                abort(http_client.NOT_FOUND, e.message)
                 return
 
-            LOG.debug('DELETE /keys/ lookup with name=%s found object: %s', name, kvp_db)
+            kvp_db = KeyValuePairAPI.to_model(kvp_api)
+
+            LOG.debug('DELETE /keys/ lookup with scope=%s name=%s found object: %s',
+                      scope, name, kvp_db)
 
             try:
                 KeyValuePair.delete(kvp_db)
@@ -144,12 +191,18 @@ def delete(self, name):
         extra = {'kvp_db': kvp_db}
         LOG.audit('KeyValuePair deleted. KeyValuePair.id=%s' % (kvp_db.id), extra=extra)
 
-    def _get_lock_name_for_key(self, name):
+    def _get_lock_name_for_key(self, name, scope=SYSTEM_SCOPE):
         """
         Retrieve a coordination lock name for the provided datastore item name.
 
         :param name: Datastore item name (PK).
         :type name: ``str``
         """
-        lock_name = 'kvp-crud-%s' % (name)
+        lock_name = 'kvp-crud-%s.%s' % (scope, name)
         return lock_name
+
+    def _validate_scope(self, scope):
+        if scope not in ALLOWED_SCOPES:
+            msg = 'Scope %s is not in allowed scopes list: %s.' % (scope, ALLOWED_SCOPES)
+            abort(http_client.BAD_REQUEST, msg)
+            return