Fix make-barrier-cost behavior when no constraints are passed (#38)

* fixing optimization import, update version to 0.7.2, update changelog * hotfix to make-barrier-qoi: can accept no constraints and applies np.atleast_2d toqoi * add fix to normilization in qoi.py and utils.stack_array * revert change to test_learn.py test - doesn't use utils.stacke_array * revise stack array test * update changelog * python black commands directory * fix typo in changelog and remove unused imports in ml.learn/learn_alt * applied isort * applied black==22.8.0, and add black version to dev deps Co-authored-by: Ryan <[email protected]>
LLNL · Sep 19, 2022 · 00451ee · 00451ee
1 parent 4f7312b
commit 00451ee
Show file tree

Hide file tree

Showing 23 changed files with 69 additions and 188 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,14 @@ All notable changes to Merlin Spellbook will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.3]
+### Fixed
+- `make-barrier-qoi` can accept no constraints.
+- `make-barrier-qoi` applies `numpy.atleast_2d` to the qoi before returning.
+- `make-barrier-qoi` correctly normalizes inputs from 0 to 1.
+- `utils.stack_arrays` returns inputs in correct shape: (# of features, dim of feature).
+- `ml.learn` and `ml.learn_alt` use correct `load_infiles` function from `utils`.
+
 ## [0.7.2]
 
 ### Fixed

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -1,6 +1,6 @@
 # Development dependencies.
 pylint
-black
+black==22.8.0
 dep-license
 flake8
 isort

diff --git a/setup.py b/setup.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin-Spellbook, Version: 0.7.2.
+# This file is part of Merlin-Spellbook, Version: 0.7.3.
 #
 # For details, see https://github.com/LLNL/merlin-spellbook.
 #
@@ -58,10 +58,7 @@ def _pip_requirement(req):
 def _reqs(*f):
     return [
         _pip_requirement(r)
-        for r in (
-            _strip_comments(line)
-            for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines()
-        )
+        for r in (_strip_comments(line) for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines())
         if r
     ]
 

diff --git a/spellbook/__init__.py b/spellbook/__init__.py
@@ -1,3 +1,3 @@
-__version__ = "0.7.2"
+__version__ = "0.7.3"
 
 VERSION = __version__
diff --git a/spellbook/commands/collect.py b/spellbook/commands/collect.py
@@ -11,13 +11,7 @@
     type=str,
     help="whitespace separated list of files to collect",
 )
-@click.option(
-    "-outfile",
-    required=False,
-    default="results.hdf5",
-    type=str,
-    help="output file",
-)
+@click.option("-outfile", required=False, default="results.hdf5", type=str, help="output file")
 def cli(instring, outfile):
     """
     Collect many json files into a single json file

diff --git a/spellbook/commands/learn.py b/spellbook/commands/learn.py
@@ -45,13 +45,5 @@ def cli(infile, x, y, outfile, regressor):
     """
     from spellbook.ml import learn_alt as learn
 
-    args = SimpleNamespace(
-        **{
-            "infile": infile,
-            "X": x,
-            "y": y,
-            "outfile": outfile,
-            "regressor": regressor,
-        }
-    )
+    args = SimpleNamespace(**{"infile": infile, "X": x, "y": y, "outfile": outfile, "regressor": regressor})
     learn.random_forest(args)
diff --git a/spellbook/commands/make-samples.py b/spellbook/commands/make-samples.py
@@ -9,20 +9,8 @@
     type=int,
     help="random number seed for generating samples",
 )
-@click.option(
-    "-n",
-    required=False,
-    default=100,
-    type=int,
-    help="number of samples",
-)
-@click.option(
-    "-dims",
-    required=False,
-    default=2,
-    type=int,
-    help="number of dimensions",
-)
+@click.option("-n", required=False, default=100, type=int, help="number of samples")
+@click.option("-dims", required=False, default=2, type=int, help="number of dimensions")
 @click.option(
     "-sample_type",
     required=False,

diff --git a/spellbook/commands/serialize.py b/spellbook/commands/serialize.py
@@ -6,13 +6,7 @@
 
 
 @click.command()
-@click.option(
-    "--output",
-    required=False,
-    default="output.json",
-    type=str,
-    help="output file",
-)
+@click.option("--output", required=False, default="output.json", type=str, help="output file")
 @click.option(
     "--vars",
     required=False,

diff --git a/spellbook/commands/stack-npz.py b/spellbook/commands/stack-npz.py
@@ -13,17 +13,8 @@
     type=bool,
     help="Force write the target file",
 )
-@click.argument(
-    "target",
-    required=True,
-    type=str,
-)
-@click.argument(
-    "source",
-    required=True,
-    type=str,
-    nargs=-1,
-)
+@click.argument("target", required=True, type=str)
+@click.argument("source", required=True, type=str, nargs=-1)
 def cli(force, target, source):
     """
     stacker for npz files.

diff --git a/spellbook/data_formatting/conduit/python/conduit_bundler.py b/spellbook/data_formatting/conduit/python/conduit_bundler.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-<PENDING>
 # All rights reserved.
-# This file is part of merlin-spellbook, Version: 0.7.2.
+# This file is part of merlin-spellbook, Version: 0.7.3.
 #
 # For details, see https://github.com/LLNL/merlin-spellbook.
 #
@@ -58,9 +58,7 @@ def determine_protocol(fname):
     if ext.startswith("."):
         protocol = ext.lower().strip(".")
     else:
-        raise ValueError(
-            "{0} needs an ext (eg .hdf5) to determine protocol!".format(fname)
-        )
+        raise ValueError("{0} needs an ext (eg .hdf5) to determine protocol!".format(fname))
     # Map .h5 to .hdf5
     if protocol == "h5":
         protocol = "hdf5"
@@ -157,10 +155,7 @@ def dump_node(
         try:
             conduit.relay.io.save(conduit_node, fname, options=save_options)
         except TypeError:  # Conduit version needs to be updated.
-            LOG.error(
-                "Unable to customize save: please upgrade conduit to "
-                "expose save options!"
-            )
+            LOG.error("Unable to customize save: please upgrade conduit to " "expose save options!")
             conduit.relay.io.save(conduit_node, fname)
     else:
         conduit.relay.io.save(conduit_node, fname)

diff --git a/spellbook/data_formatting/conduit/python/translator.py b/spellbook/data_formatting/conduit/python/translator.py
@@ -86,9 +86,7 @@ def run(_input, output, schema):
             if data_loader.has_path(sample_path):
                 data_loader.read(filtered_node[path], sample_path)
             else:
-                filtered_node[
-                    sample_path
-                ] = np.nan  # if a value is missing, that could be a problem
+                filtered_node[sample_path] = np.nan  # if a value is missing, that could be a problem
         make_data_array_dict(all_dict, filtered_node)
 
     for dat in all_dict.keys():
@@ -129,9 +127,7 @@ def generate_scalar_path_pairs(node, path=""):
     children = node.child_names()
     for child in children:
         if isinstance(node[child], conduit.Node):
-            for pair in generate_scalar_path_pairs(
-                node[child], path=path + child + "/"
-            ):
+            for pair in generate_scalar_path_pairs(node[child], path=path + child + "/"):
                 yield pair
         else:
             yield path + child, node[child]

diff --git a/spellbook/data_formatting/stack_npz.py b/spellbook/data_formatting/stack_npz.py
@@ -24,9 +24,7 @@ def pad_many(arrays, dims, dont_pad_first=False, value=np.nan):
         if dont_pad_first:
             pad_dist[0] = 0
         padder = np.column_stack((zeros, pad_dist))
-        fixed.append(
-            np.pad(np.atleast_2d(a), padder, mode="constant", constant_values=value)
-        )
+        fixed.append(np.pad(np.atleast_2d(a), padder, mode="constant", constant_values=value))
     return fixed
 
 
@@ -48,11 +46,7 @@ def run(self, target, source, force=False):
 
         if not force:
             if os.path.isfile(target):
-                print(
-                    "stack_npz error opening target file (does {0} exist?).".format(
-                        target
-                    )
-                )
+                print("stack_npz error opening target file (does {0} exist?).".format(target))
                 print('Pass "-f" argument to force re-creation of output file.')
                 return
 

diff --git a/spellbook/data_formatting/translator.py b/spellbook/data_formatting/translator.py
@@ -13,12 +13,8 @@ def setup_argparse():
         help=".json file with X and y data in each sample",
         default="results.json",
     )
-    parser.add_argument(
-        "-output", help=".npz file with the arrays", default="results.npz"
-    )
-    parser.add_argument(
-        "-schema", help="schema for a single sample data", default="features.json"
-    )
+    parser.add_argument("-output", help=".npz file with the arrays", default="results.npz")
+    parser.add_argument("-schema", help="schema for a single sample data", default="features.json")
     return parser
 
 
@@ -51,9 +47,7 @@ def generate_scalar_path_pairs(node, schema, path=""):
         if child in schema.keys():
             if isinstance(node[child], dict):
                 if isinstance(schema[child], dict):
-                    for pair in generate_scalar_path_pairs(
-                        node[child], schema[child], path=path + child + "/"
-                    ):
+                    for pair in generate_scalar_path_pairs(node[child], schema[child], path=path + child + "/"):
                         yield pair
             else:
                 if not isinstance(schema[child], dict):

diff --git a/spellbook/log_formatter.py b/spellbook/log_formatter.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin Spellbook, Version: 0.7.2.
+# This file is part of Merlin Spellbook, Version: 0.7.3.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/spellbook/ml/learn.py b/spellbook/ml/learn.py
@@ -28,9 +28,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 ###############################################################################
-import numpy as np
-
 import spellbook.ml.surrogates as surrogates
+from spellbook.utils import load_infile
 
 
 try:
@@ -39,24 +38,6 @@
     import pickle
 
 
-def load_infile(args):
-    with np.load(args.infile) as data:
-        if args.X is not None:
-            X = stack_arrays(data, args.X)  # inputs
-        elif "X" in data.keys():
-            X = data["X"]
-        else:
-            X = data[data.files[0]]
-
-        if args.y is not None:
-            y = stack_arrays(data, args.y)  # outputs
-        elif "y" in data.keys():
-            y = data["y"]
-        else:
-            y = data[data.files[1]]
-    return X, y
-
-
 def make_regressor(args):
     if args.reg_args is None:
         reg_args = {}
@@ -81,8 +62,3 @@ def make_regressor(args):
     regr.fit(X, y)
     with open(args.outfile, "wb") as f:
         pickle.dump(regr, f)
-
-
-def stack_arrays(data, delimited_names, delimiter=","):
-    stacked = np.vstack([data[name] for name in delimited_names.split(delimiter)])
-    return stacked.T
diff --git a/spellbook/ml/learn_alt.py b/spellbook/ml/learn_alt.py
@@ -1,6 +1,7 @@
-import numpy as np
 from sklearn.ensemble import RandomForestRegressor
 
+from spellbook.utils import load_infile
+
 
 try:
     import cPickle as pickle
@@ -11,24 +12,6 @@
 FOREST_DEFAULTS = {"max_depth": 2, "random_state": 0, "n_estimators": 100}
 
 
-def load_infile(args):
-    with np.load(args.infile) as data:
-        if args.X is not None:
-            X = stack_arrays(data, args.X)  # inputs
-        elif "X" in data.keys():
-            X = data["X"]
-        else:
-            X = data[data.files[0]]
-
-        if args.y is not None:
-            y = stack_arrays(data, args.y)  # outputs
-        elif "y" in data.keys():
-            y = data["y"]
-        else:
-            y = data[data.files[1]]
-    return X, y
-
-
 def random_forest(args):
     forest_args = FOREST_DEFAULTS
 
@@ -50,8 +33,3 @@ def random_forest(args):
     regr.fit(X, y)
     with open(args.outfile, "wb") as f:
         pickle.dump(regr, f)
-
-
-def stack_arrays(data, delimited_names, delimiter=","):
-    stacked = np.vstack([data[name] for name in delimited_names.split(delimiter)])
-    return stacked.T
diff --git a/spellbook/ml/surrogates.py b/spellbook/ml/surrogates.py
@@ -65,11 +65,7 @@ def factory(cls, name, *args, **kwargs):
         if name in cls.all_regs:
             return cls.all_regs[name](*args, **kwargs)
         else:
-            raise ValueError(
-                "Unknown regressor name "
-                + name
-                + "! For valid choices see sklearnRegressors.names()"
-            )
+            raise ValueError("Unknown regressor name " + name + "! For valid choices see sklearnRegressors.names()")
 
     @classmethod
     def names(cls):
@@ -85,12 +81,8 @@ def test_factory():
 
 def test_random_forest():
 
-    rf1 = sklearnRegressors.factory(
-        "RandomForestRegressor", n_estimators=10, max_depth=5
-    )
-    rf2 = sklearnRegressors.factory(
-        "RandomForestRegressor", n_estimators=2, max_depth=3
-    )
+    rf1 = sklearnRegressors.factory("RandomForestRegressor", n_estimators=10, max_depth=5)
+    rf2 = sklearnRegressors.factory("RandomForestRegressor", n_estimators=2, max_depth=3)
 
     assert rf1.n_estimators == 10
     assert rf1.max_depth == 5