Merge pull request StackStorm#2631 from StackStorm/secrets_kv_store

Secrets in kv store phase #0

Makefile

@@ -319,6 +319,13 @@ packs-tests: requirements .packs-tests
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate; find ${ROOT_DIR}/contrib/* -maxdepth 0 -type d -print0 | xargs -0 -I FILENAME ./st2common/bin/st2-run-pack-tests -x -p FILENAME
 
+.PHONY: cli
+cli:
+	@echo
+	@echo "=================== Building st2 client ==================="
+	@echo
+	pushd $(CURDIR) && cd st2client && ((python setup.py develop || printf "\n\n!!! ERROR: BUILD FAILED !!!\n") || popd)
+
 .PHONY: rpms
 rpms:
 	@echo

conf/st2.dev.conf

@@ -11,6 +11,9 @@ mask_secrets = False
 # Development vagrant VM is setup to run on 172.168.50.50.
 allow_origin = http://172.168.50.50:3000
 
+[keyvalue]
+encryption_key_path = conf/st2_kvstore_demo.crypto.key.json
+
 [stream]
 # Host and port to bind the API server.
 host = 0.0.0.0

conf/st2_kvstore_demo.crypto.key.json

@@ -0,0 +1 @@
+{"hmacKey": {"hmacKeyString": "685EWJ8U2U_c3Ulv9ibB6aa0Y9aYm-0bCwe2mqjLJuw", "size": 256}, "aesKeyString": "eKGFrfNEUhAgX0uofaU1wQ", "mode": "CBC", "size": 128}

st2api/st2api/controllers/v1/keyvalue.py

@@ -19,6 +19,7 @@
 from mongoengine import ValidationError
 
 from st2common import log as logging
+from st2common.exceptions.keyvalue import CryptoKeyNotSetupException
 from st2common.models.api.keyvalue import KeyValuePairAPI
 from st2common.models.api.base import jsexpose
 from st2common.persistence.keyvalue import KeyValuePair
@@ -44,14 +45,20 @@ def __init__(self):
         self._coordinator = coordination.get_coordinator()
         self.get_one_db_method = self.__get_by_name
 
-    @jsexpose(arg_types=[str])
-    def get_one(self, name):
+    @jsexpose(arg_types=[str, str])
+    def get_one(self, name, decrypt='false'):
         """
             List key by name.
 
             Handle:
                 GET /keys/key1
         """
+
+        if not decrypt:
+            decrypt = False
+        else:
+            decrypt = (decrypt == 'true' or decrypt == 'True' or decrypt == '1')
+
         kvp_db = self.__get_by_name(name=name)
 
         if not kvp_db:
@@ -60,7 +67,7 @@ def get_one(self, name):
             return
 
         try:
-            kvp_api = KeyValuePairAPI.from_model(kvp_db)
+            kvp_api = KeyValuePairAPI.from_model(kvp_db, mask_secrets=(not decrypt))
         except (ValidationError, ValueError) as e:
             abort(http_client.INTERNAL_SERVER_ERROR, str(e))
             return
@@ -78,12 +85,21 @@ def get_all(self, **kw):
         # Prefix filtering
         prefix_filter = kw.get('prefix', None)
 
+        decrypt = kw.get('decrypt', None)
+        if not decrypt:
+            decrypt = False
+        else:
+            decrypt = (decrypt == 'true' or decrypt == 'True' or decrypt == '1')
+            del kw['decrypt']
+
         if prefix_filter:
             kw['name__startswith'] = prefix_filter
             del kw['prefix']
 
         kvp_dbs = KeyValuePair.get_all(**kw)
-        kvps = [KeyValuePairAPI.from_model(kvp_db) for kvp_db in kvp_dbs]
+        kvps = [KeyValuePairAPI.from_model(
+            kvp_db, mask_secrets=(not decrypt)) for kvp_db in kvp_dbs
+        ]
 
         return kvps
 
@@ -113,7 +129,10 @@ def put(self, name, kvp):
                 LOG.exception('Validation failed for key value data=%s', kvp)
                 abort(http_client.BAD_REQUEST, str(e))
                 return
-
+            except CryptoKeyNotSetupException as e:
+                LOG.exception(str(e))
+                abort(http_client.BAD_REQUEST, str(e))
+                return
         extra = {'kvp_db': kvp_db}
         LOG.audit('KeyValuePair updated. KeyValuePair.id=%s' % (kvp_db.id), extra=extra)
 

st2api/tests/unit/controllers/v1/test_kvps.py

@@ -31,6 +31,12 @@
     'ttl': 10
 }
 
+SECRET_KVP = {
+    'name': 'secret_key1',
+    'value': 'secret_value1',
+    'secret': True
+}
+
 
 class TestKeyValuePairController(FunctionalTest):
 
@@ -86,6 +92,52 @@ def test_put_with_ttl(self):
         self.assertTrue(get_resp.json[0]['expire_timestamp'])
         self.__do_delete(self.__get_kvp_id(put_resp))
 
+    def test_put_secret(self):
+        put_resp = self.__do_put('secret_key1', SECRET_KVP)
+        kvp_id = self.__get_kvp_id(put_resp)
+        get_resp = self.__do_get_one(kvp_id)
+        self.assertTrue(get_resp.json['encrypted'])
+        crypto_val = get_resp.json['value']
+        self.assertNotEqual(SECRET_KVP['value'], crypto_val)
+        self.__do_delete(self.__get_kvp_id(put_resp))
+
+    def test_get_one_secret_no_decrypt(self):
+        put_resp = self.__do_put('secret_key1', SECRET_KVP)
+        kvp_id = self.__get_kvp_id(put_resp)
+        get_resp = self.app.get('/v1/keys/secret_key1')
+        self.assertEqual(get_resp.status_int, 200)
+        self.assertEqual(self.__get_kvp_id(get_resp), kvp_id)
+        self.assertTrue(get_resp.json['secret'])
+        self.assertTrue(get_resp.json['encrypted'])
+        self.__do_delete(kvp_id)
+
+    def test_get_one_secret_decrypt(self):
+        put_resp = self.__do_put('secret_key1', SECRET_KVP)
+        kvp_id = self.__get_kvp_id(put_resp)
+        get_resp = self.app.get('/v1/keys/secret_key1?decrypt=true')
+        self.assertEqual(get_resp.status_int, 200)
+        self.assertEqual(self.__get_kvp_id(get_resp), kvp_id)
+        self.assertTrue(get_resp.json['secret'])
+        self.assertFalse(get_resp.json['encrypted'])
+        self.assertEqual(get_resp.json['value'], SECRET_KVP['value'])
+        self.__do_delete(kvp_id)
+
+    def test_get_all_decrypt(self):
+        put_resp = self.__do_put('secret_key1', SECRET_KVP)
+        kvp_id_1 = self.__get_kvp_id(put_resp)
+        put_resp = self.__do_put('key1', KVP)
+        kvp_id_2 = self.__get_kvp_id(put_resp)
+        kvps = {'key1': KVP, 'secret_key1': SECRET_KVP}
+        stored_kvps = self.app.get('/v1/keys?decrypt=true').json
+        self.assertTrue(len(stored_kvps), 2)
+        for stored_kvp in stored_kvps:
+            self.assertFalse(stored_kvp['encrypted'])
+            exp_kvp = kvps.get(stored_kvp['name'])
+            self.assertTrue(exp_kvp is not None)
+            self.assertEqual(exp_kvp['value'], stored_kvp['value'])
+        self.__do_delete(kvp_id_1)
+        self.__do_delete(kvp_id_2)
+
     def test_put_delete(self):
         put_resp = self.__do_put('key1', KVP)
         self.assertEqual(put_resp.status_int, 200)

st2client/st2client/commands/keyvalue.py

@@ -57,7 +57,7 @@ def __init__(self, description, app, subparsers, parent_parser=None):
 
 
 class KeyValuePairListCommand(resource.ResourceListCommand):
-    display_attributes = ['name', 'value', 'expire_timestamp']
+    display_attributes = ['name', 'value', 'secret', 'encrypted', 'expire_timestamp']
     attribute_transform_functions = {
         'expire_timestamp': format_isodate_for_user_timezone,
     }
@@ -68,11 +68,16 @@ def __init__(self, *args, **kwargs):
         # Filter options
         self.parser.add_argument('--prefix', help=('Only return values which name starts with the '
                                                    ' provided prefix.'))
+        self.parser.add_argument('--decrypt', action='store_true',
+                                 help='Decrypt secrets and display plain text.')
 
     def run_and_print(self, args, **kwargs):
         if args.prefix:
             kwargs['prefix'] = args.prefix
 
+        decrypt = getattr(args, 'decrypt', False)
+        kwargs['params'] = {'decrypt': str(decrypt).lower()}
+
         instances = self.run(args, **kwargs)
         self.print_output(reversed(instances), table.MultiColumnTable,
                           attributes=args.attr, widths=args.width,
@@ -82,7 +87,19 @@ def run_and_print(self, args, **kwargs):
 
 class KeyValuePairGetCommand(resource.ResourceGetCommand):
     pk_argument_name = 'name'
-    display_attributes = ['name', 'value', 'expire_timestamp']
+    display_attributes = ['name', 'value', 'secret', 'encrypted', 'expire_timestamp']
+
+    def __init__(self, kv_resource, *args, **kwargs):
+        super(KeyValuePairGetCommand, self).__init__(kv_resource, *args, **kwargs)
+        self.parser.add_argument('-d', '--decrypt', action='store_true',
+                                 help='Decrypt secret if encrypted and show plain text.')
+
+    @resource.add_auth_token_to_kwargs_from_cli
+    def run(self, args, **kwargs):
+        resource_name = getattr(args, self.pk_argument_name, None)
+        decrypt = getattr(args, 'decrypt', False)
+        kwargs['params'] = {'decrypt': str(decrypt).lower()}
+        return self.get_resource_by_id(id=resource_name, **kwargs)
 
 
 class KeyValuePairSetCommand(resource.ResourceCommand):
@@ -101,6 +118,9 @@ def __init__(self, resource, *args, **kwargs):
         self.parser.add_argument('value', help='Value paired with the key.')
         self.parser.add_argument('-l', '--ttl', dest='ttl', type=int, default=None,
                                  help='TTL (in seconds) for this value.')
+        self.parser.add_argument('-e', '--encrypt', dest='secret',
+                                 action='store_true',
+                                 help='Encrypt value before saving the value.')
 
     @add_auth_token_to_kwargs_from_cli
     def run(self, args, **kwargs):
@@ -109,6 +129,9 @@ def run(self, args, **kwargs):
         instance.name = args.name
         instance.value = args.value
 
+        if args.secret:
+            instance.secret = args.secret
+
         if args.ttl:
             instance.ttl = args.ttl
 

st2client/st2client/commands/resource.py

@@ -287,7 +287,7 @@ def __init__(self, resource, *args, **kwargs):
     @add_auth_token_to_kwargs_from_cli
     def run(self, args, **kwargs):
         resource_id = getattr(args, self.pk_argument_name, None)
-        return self.get_resource(resource_id, **kwargs)
+        return self.get_resource_by_id(resource_id, **kwargs)
 
     def run_and_print(self, args, **kwargs):
         try:

st2client/st2client/models/core.py

@@ -169,7 +169,7 @@ def get_all(self, **kwargs):
         prefix = kwargs.pop('prefix', None)
         user = kwargs.pop('user', None)
 
-        params = {}
+        params = kwargs.pop('params', {})
         if limit and limit <= 0:
             limit = None
         if limit:
@@ -240,7 +240,8 @@ def query(self, **kwargs):
         if 'limit' in kwargs and kwargs.get('limit') <= 0:
             kwargs.pop('limit')
         token = kwargs.get('token', None)
-        params = {}
+        params = kwargs.get('params', {})
+        print(params)
         for k, v in six.iteritems(kwargs):
             if k != 'token':
                 params[k] = v

st2client/tests/unit/test_commands.py

@@ -72,23 +72,9 @@ def test_command_get_by_id(self):
         expected = json.loads(json.dumps(base.RESOURCES[0]))
         self.assertEqual(actual, expected)
 
-    @mock.patch.object(
-        models.ResourceManager, 'get_by_name',
-        mock.MagicMock(return_value=base.FakeResource(**base.RESOURCES[0])))
-    @mock.patch.object(
-        models.ResourceManager, 'get_by_id',
-        mock.MagicMock(return_value=None))
-    def test_command_get_by_name(self):
-        args = self.parser.parse_args(['fakeresource', 'get', 'abc'])
-        self.assertEqual(args.func, self.branch.commands['get'].run_and_print)
-        instance = self.branch.commands['get'].run(args)
-        actual = instance.serialize()
-        expected = json.loads(json.dumps(base.RESOURCES[0]))
-        self.assertEqual(actual, expected)
-
     @mock.patch.object(
         httpclient.HTTPClient, 'get',
-        mock.MagicMock(return_value=base.FakeResponse(json.dumps([base.RESOURCES[0]]), 200, 'OK')))
+        mock.MagicMock(return_value=base.FakeResponse(json.dumps(base.RESOURCES[0]), 200, 'OK')))
     def test_command_get(self):
         args = self.parser.parse_args(['fakeresource', 'get', 'abc'])
         self.assertEqual(args.func, self.branch.commands['get'].run_and_print)

st2common/in-requirements.txt

@@ -11,6 +11,7 @@ oslo.config
 paramiko
 pyyaml
 pymongo
+python-keyczar
 requests
 retrying
 semver

st2common/st2common/config.py

@@ -156,6 +156,17 @@ def register_opts(ignore_errors=False):
     ]
     do_register_opts(api_opts, 'api', ignore_errors)
 
+    # Key Value store options
+    keyvalue_opts = [
+        cfg.BoolOpt('enable_encryption', default=True,
+                    help='Allow encryption of values in key value stored qualified as "secret".'),
+        cfg.StrOpt('encryption_key_path', default='',
+                   help='Location of the symmetric encryption key for encrypting values in ' +
+                        'kvstore. This key should be in JSON and should\'ve been ' +
+                        'generated using keyczar.')
+    ]
+    do_register_opts(keyvalue_opts, group='keyvalue')
+
     # Common auth options
     auth_opts = [
         cfg.StrOpt('api_url', default=None,

st2common/st2common/exceptions/keyvalue.py

@@ -0,0 +1,24 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from st2common.exceptions import StackStormBaseException
+
+__all__ = [
+    'CryptoKeyNotSetupException',
+]
+
+
+class CryptoKeyNotSetupException(StackStormBaseException):
+    pass