From 3b476b29aab7f28015f84a21dbe0903bbfe4bcd5 Mon Sep 17 00:00:00 2001 From: John Andersen Date: Thu, 30 Apr 2020 10:11:07 -0700 Subject: [PATCH] examples: Import from top level Signed-off-by: John Andersen --- docs/quickstart/model.rst | 2 +- docs/tutorials/operations.rst | 8 ++-- examples/MNIST/test_mnist.py | 6 +-- examples/io/io_usage.py | 13 +------ examples/maintained/demoapp/source.py | 39 +++++-------------- examples/model/slr/slr.py | 3 +- examples/model/slr/test_slr.py | 2 +- examples/quickstart_async.py | 2 +- examples/shouldi/shouldi/bandit.py | 4 +- examples/shouldi/shouldi/cargo_audit.py | 3 +- examples/shouldi/shouldi/dependency_check.py | 3 +- examples/shouldi/shouldi/golangci_lint.py | 3 +- examples/shouldi/shouldi/npm_audit.py | 4 +- examples/shouldi/shouldi/pypi.py | 3 +- examples/shouldi/shouldi/safety.py | 4 +- examples/shouldi/tests/test_bandit.py | 2 +- examples/shouldi/tests/test_cargo_audit.py | 8 ++-- examples/shouldi/tests/test_cli.py | 2 +- .../shouldi/tests/test_dependency_check.py | 8 ++-- examples/shouldi/tests/test_golangci_lint.py | 8 ++-- examples/shouldi/tests/test_npm_audit.py | 8 ++-- examples/shouldi/tests/test_pypi.py | 2 +- examples/shouldi/tests/test_safety.py | 2 +- examples/source/custom_sqlite.py | 20 ++-------- examples/source/test_custom_sqlite.py | 3 +- examples/test_quickstart.py | 3 +- 26 files changed, 62 insertions(+), 103 deletions(-) diff --git a/docs/quickstart/model.rst b/docs/quickstart/model.rst index f91fb1f2a0..28e68c4df8 100644 --- a/docs/quickstart/model.rst +++ b/docs/quickstart/model.rst @@ -208,7 +208,7 @@ The ouput should be as follows {'Years': 7, 'Expertise': 15, 'Trust': 0.8, 'Salary': 80.0} Check out the plugin docs for :doc:`/plugins/dffml_model` for usage of other -models. The :doc:`API ` docs may also be useful. +models. The :doc:`API ` docs may also be useful. Data Sources ------------ diff --git a/docs/tutorials/operations.rst b/docs/tutorials/operations.rst index 9ec16151ca..5d0bec7ccf 100644 --- a/docs/tutorials/operations.rst +++ b/docs/tutorials/operations.rst @@ -371,7 +371,7 @@ Let's write an operation to grab the JSON information about a package. **shouldi/pypi.py** .. literalinclude:: /../examples/shouldi/shouldi/pypi.py - :lines: 1-35 + :lines: 1-34 After we have the package information, we extract the version and URL where we can get the source code. @@ -379,7 +379,7 @@ can get the source code. **shouldi/pypi.py** .. literalinclude:: /../examples/shouldi/shouldi/pypi.py - :lines: 38-60 + :lines: 37-59 Once we have the URL, we download the package source and extract it to a temporary directory. @@ -387,7 +387,7 @@ temporary directory. **shouldi/pypi.py** .. literalinclude:: /../examples/shouldi/shouldi/pypi.py - :lines: 63-82 + :lines: 62-81 Finally, we make a ``cleanup`` operation to remove the directory once we're done with it. @@ -395,7 +395,7 @@ with it. **shouldi/pypi.py** .. literalinclude:: /../examples/shouldi/shouldi/pypi.py - :lines: 85-90 + :lines: 84-89 Now we write tests for each operation. diff --git a/examples/MNIST/test_mnist.py b/examples/MNIST/test_mnist.py index 7434680334..697a06fd25 100644 --- a/examples/MNIST/test_mnist.py +++ b/examples/MNIST/test_mnist.py @@ -2,14 +2,14 @@ import ast import sys import json +import shutil +import pathlib import tempfile import contextlib import subprocess import unittest.mock -import shutil -import pathlib -from dffml.util.os import chdir +from dffml import chdir def sh_filepath(filename): diff --git a/examples/io/io_usage.py b/examples/io/io_usage.py index cf9c6f123a..035ffdb63b 100644 --- a/examples/io/io_usage.py +++ b/examples/io/io_usage.py @@ -1,14 +1,5 @@ import asyncio - -from dffml import train, Features, DefFeature -from dffml.operation.output import GetSingle -from dffml.df.memory import MemoryOrchestrator -from dffml.operation.mapping import create_mapping -from dffml.operation.preprocess import literal_eval -from dffml.df.types import DataFlow, Input, InputFlow -from dffml.operation.io import AcceptUserInput, print_output -from dffml.operation.model import model_predict, ModelPredictConfig -from dffml.model.slr import SLRModel +from dffml import * slr_model = SLRModel( features=Features(DefFeature("Years", int, 1),), @@ -62,7 +53,7 @@ async def main(): - # train the model + # Train the model await train( slr_model, {"Years": 0, "Salary": 10}, diff --git a/examples/maintained/demoapp/source.py b/examples/maintained/demoapp/source.py index d7ae275cd5..ab890f280b 100644 --- a/examples/maintained/demoapp/source.py +++ b/examples/maintained/demoapp/source.py @@ -2,19 +2,16 @@ import aiomysql from typing import AsyncIterator, NamedTuple, Dict -from dffml.base import BaseConfig -from dffml.record import Record -from dffml.source.source import BaseSourceContext, BaseSource -from dffml.util.cli.arg import Arg -from dffml.util.entrypoint import entrypoint +from dffml import config, entrypoint, Record, BaseSourceContext, BaseSource -class DemoAppSourceConfig(BaseConfig, NamedTuple): - host: str - port: int - user: str - password: str - db: str +@config +class DemoAppSourceConfig: + host: str = "127.0.0.1" + port: int = 3306 + user: str = "user" + password: str = "pass" + db: str = "db" class DemoAppSourceContext(BaseSourceContext): @@ -69,6 +66,7 @@ async def __aexit__(self, exc_type, exc_value, traceback): @entrypoint("demoapp") class DemoAppSource(BaseSource): + CONFIG = DemoAppSourceConfig CONTEXT = DemoAppSourceContext async def __aenter__(self) -> "DemoAppSource": @@ -87,22 +85,3 @@ async def __aexit__(self, exc_type, exc_value, traceback): await self.__db.__aexit__(exc_type, exc_value, traceback) self.pool.close() await self.pool.wait_closed() - - @classmethod - def args(cls, args, *above) -> Dict[str, Arg]: - cls.config_set(args, above, "host", Arg(default="127.0.0.1")) - cls.config_set(args, above, "port", Arg(type=int, default=3306)) - cls.config_set(args, above, "user", Arg(default="user")) - cls.config_set(args, above, "password", Arg(default="pass")) - cls.config_set(args, above, "db", Arg(default="db")) - return args - - @classmethod - def config(cls, config, *above): - return DemoAppSourceConfig( - host=cls.config_get(config, above, "host"), - port=cls.config_get(config, above, "port"), - user=cls.config_get(config, above, "user"), - password=cls.config_get(config, above, "password"), - db=cls.config_get(config, above, "db"), - ) diff --git a/examples/model/slr/slr.py b/examples/model/slr/slr.py index 2e7be4c8e3..7e0f8845bf 100644 --- a/examples/model/slr/slr.py +++ b/examples/model/slr/slr.py @@ -1,6 +1,5 @@ -from dffml import Features, DefFeature +from dffml import Features, DefFeature, SLRModel from dffml.noasync import train, accuracy, predict -from dffml.model.slr import SLRModel model = SLRModel( features=Features(DefFeature("f1", float, 1)), diff --git a/examples/model/slr/test_slr.py b/examples/model/slr/test_slr.py index e75460d9a6..b8216a510e 100644 --- a/examples/model/slr/test_slr.py +++ b/examples/model/slr/test_slr.py @@ -7,7 +7,7 @@ import subprocess import unittest.mock -from dffml.util.os import chdir +from dffml import chdir def sh_filepath(filename): diff --git a/examples/quickstart_async.py b/examples/quickstart_async.py index b8c616b83b..bbd0e86f08 100644 --- a/examples/quickstart_async.py +++ b/examples/quickstart_async.py @@ -1,7 +1,7 @@ import asyncio -from dffml_model_scikit import LinearRegressionModel from dffml import train, accuracy, predict, Features, DefFeature +from dffml_model_scikit import LinearRegressionModel async def main(): diff --git a/examples/shouldi/shouldi/bandit.py b/examples/shouldi/shouldi/bandit.py index 538c0fda81..b1c529c33a 100644 --- a/examples/shouldi/shouldi/bandit.py +++ b/examples/shouldi/shouldi/bandit.py @@ -3,8 +3,8 @@ import asyncio from typing import Dict, Any -from dffml.df.base import op -from dffml.df.types import Definition +from dffml import op +from dffml import Definition package_src_dir = Definition(name="package_src_dir", primitive="str") bandit_output = Definition(name="bandit_output", primitive="Dict[str, Any]") diff --git a/examples/shouldi/shouldi/cargo_audit.py b/examples/shouldi/shouldi/cargo_audit.py index 6646f9f283..a9eb2110f8 100644 --- a/examples/shouldi/shouldi/cargo_audit.py +++ b/examples/shouldi/shouldi/cargo_audit.py @@ -2,8 +2,7 @@ import asyncio from typing import Dict, Any -from dffml.df.base import op -from dffml.df.types import Definition +from dffml import op, Definition package_src_dir = Definition(name="package_src_dir", primitive="str") cargo_audit_output = Definition( diff --git a/examples/shouldi/shouldi/dependency_check.py b/examples/shouldi/shouldi/dependency_check.py index 82570328c9..f6953da1ba 100644 --- a/examples/shouldi/shouldi/dependency_check.py +++ b/examples/shouldi/shouldi/dependency_check.py @@ -5,8 +5,7 @@ import asyncio from typing import Dict, Any -from dffml.df.base import op -from dffml.df.types import Definition +from dffml import op, Definition package_src_dir = Definition(name="package_src_dir", primitive="str") dependency_check_output = Definition( diff --git a/examples/shouldi/shouldi/golangci_lint.py b/examples/shouldi/shouldi/golangci_lint.py index 69af07ba87..92d227dd5c 100644 --- a/examples/shouldi/shouldi/golangci_lint.py +++ b/examples/shouldi/shouldi/golangci_lint.py @@ -2,8 +2,7 @@ import asyncio from typing import Dict, Any -from dffml.df.base import op -from dffml.df.types import Definition +from dffml import op, Definition package_src_dir = Definition(name="package_src_dir", primitive="str") golangci_lint_output = Definition( diff --git a/examples/shouldi/shouldi/npm_audit.py b/examples/shouldi/shouldi/npm_audit.py index 62a4b2ab15..0a034dbd4f 100644 --- a/examples/shouldi/shouldi/npm_audit.py +++ b/examples/shouldi/shouldi/npm_audit.py @@ -2,8 +2,8 @@ import asyncio from typing import Dict, Any -from dffml.df.base import op -from dffml.df.types import Definition +from dffml import op +from dffml import Definition package_src_dir = Definition(name="package_src_dir", primitive="str") npm_audit_output = Definition( diff --git a/examples/shouldi/shouldi/pypi.py b/examples/shouldi/shouldi/pypi.py index c4f0e6f423..f2d69fe87e 100644 --- a/examples/shouldi/shouldi/pypi.py +++ b/examples/shouldi/shouldi/pypi.py @@ -4,8 +4,7 @@ import aiohttp -from dffml.df.base import op -from dffml.df.types import Definition, Stage +from dffml import op, Definition, Stage from .safety import package, package_version from .bandit import package_src_dir diff --git a/examples/shouldi/shouldi/safety.py b/examples/shouldi/shouldi/safety.py index b200d48232..0fdadabf2d 100644 --- a/examples/shouldi/shouldi/safety.py +++ b/examples/shouldi/shouldi/safety.py @@ -3,8 +3,8 @@ import asyncio from typing import Dict, Any -from dffml.df.base import op -from dffml.df.types import Definition +from dffml import op +from dffml import Definition package = Definition(name="package", primitive="str") package_version = Definition(name="package_version", primitive="str") diff --git a/examples/shouldi/tests/test_bandit.py b/examples/shouldi/tests/test_bandit.py index 873885da6b..d905f200c4 100644 --- a/examples/shouldi/tests/test_bandit.py +++ b/examples/shouldi/tests/test_bandit.py @@ -1,6 +1,6 @@ import os -from dffml.util.asynctestcase import AsyncTestCase +from dffml import AsyncTestCase from shouldi.bandit import run_bandit diff --git a/examples/shouldi/tests/test_cargo_audit.py b/examples/shouldi/tests/test_cargo_audit.py index 55c88cc851..e9f47b8133 100644 --- a/examples/shouldi/tests/test_cargo_audit.py +++ b/examples/shouldi/tests/test_cargo_audit.py @@ -1,8 +1,10 @@ import pathlib -from dffml.util.os import prepend_to_path -from dffml.util.net import cached_download_unpack_archive -from dffml.util.asynctestcase import AsyncTestCase +from dffml import ( + prepend_to_path, + cached_download_unpack_archive, + AsyncTestCase, +) from shouldi.cargo_audit import run_cargo_audit, run_cargo_build diff --git a/examples/shouldi/tests/test_cli.py b/examples/shouldi/tests/test_cli.py index f0d7f30e46..a95437b10a 100644 --- a/examples/shouldi/tests/test_cli.py +++ b/examples/shouldi/tests/test_cli.py @@ -1,7 +1,7 @@ import io from unittest.mock import patch -from dffml.util.asynctestcase import AsyncTestCase +from dffml import AsyncTestCase from shouldi.cli import ShouldI diff --git a/examples/shouldi/tests/test_dependency_check.py b/examples/shouldi/tests/test_dependency_check.py index d9ac6b3740..263773eb5f 100644 --- a/examples/shouldi/tests/test_dependency_check.py +++ b/examples/shouldi/tests/test_dependency_check.py @@ -1,8 +1,10 @@ import pathlib -from dffml.util.os import prepend_to_path -from dffml.util.net import cached_download_unpack_archive -from dffml.util.asynctestcase import AsyncTestCase +from dffml import ( + prepend_to_path, + cached_download_unpack_archive, + AsyncTestCase, +) from shouldi.dependency_check import run_dependency_check diff --git a/examples/shouldi/tests/test_golangci_lint.py b/examples/shouldi/tests/test_golangci_lint.py index f4bfab611e..f4b0db7a7f 100644 --- a/examples/shouldi/tests/test_golangci_lint.py +++ b/examples/shouldi/tests/test_golangci_lint.py @@ -2,9 +2,11 @@ import copy import pathlib -from dffml.util.os import prepend_to_path -from dffml.util.net import cached_download_unpack_archive -from dffml.util.asynctestcase import AsyncTestCase +from dffml import ( + prepend_to_path, + cached_download_unpack_archive, + AsyncTestCase, +) from shouldi.golangci_lint import run_golangci_lint diff --git a/examples/shouldi/tests/test_npm_audit.py b/examples/shouldi/tests/test_npm_audit.py index 2def0c9d56..c092a8018d 100644 --- a/examples/shouldi/tests/test_npm_audit.py +++ b/examples/shouldi/tests/test_npm_audit.py @@ -1,8 +1,10 @@ import pathlib -from dffml.util.os import prepend_to_path -from dffml.util.net import cached_download_unpack_archive -from dffml.util.asynctestcase import AsyncTestCase +from dffml import ( + prepend_to_path, + cached_download_unpack_archive, + AsyncTestCase, +) from shouldi.npm_audit import run_npm_audit diff --git a/examples/shouldi/tests/test_pypi.py b/examples/shouldi/tests/test_pypi.py index 7fed9f3cfa..5441a0bb8d 100644 --- a/examples/shouldi/tests/test_pypi.py +++ b/examples/shouldi/tests/test_pypi.py @@ -2,7 +2,7 @@ import shutil import tempfile -from dffml.util.asynctestcase import AsyncTestCase +from dffml import AsyncTestCase from shouldi.pypi import pypi_package_json from shouldi.pypi import pypi_latest_package_version diff --git a/examples/shouldi/tests/test_safety.py b/examples/shouldi/tests/test_safety.py index 330466f37c..824b3c37f6 100644 --- a/examples/shouldi/tests/test_safety.py +++ b/examples/shouldi/tests/test_safety.py @@ -1,4 +1,4 @@ -from dffml.util.asynctestcase import AsyncTestCase +from dffml import AsyncTestCase from shouldi.safety import safety_check diff --git a/examples/source/custom_sqlite.py b/examples/source/custom_sqlite.py index 7e5f2e4060..1d4aed59f1 100644 --- a/examples/source/custom_sqlite.py +++ b/examples/source/custom_sqlite.py @@ -2,13 +2,11 @@ from collections import OrderedDict from typing import AsyncIterator, NamedTuple, Dict -from dffml.base import BaseConfig -from dffml.record import Record -from dffml.source.source import BaseSourceContext, BaseSource -from dffml.util.cli.arg import Arg +from dffml import config, Record, BaseSource, BaseSourceContext -class CustomSQLiteSourceConfig(BaseConfig, NamedTuple): +@config +class CustomSQLiteSourceConfig: filename: str @@ -77,6 +75,7 @@ async def __aexit__(self, exc_type, exc_value, traceback): class CustomSQLiteSource(BaseSource): + CONFIG = CustomSQLiteSourceConfig CONTEXT = CustomSQLiteSourceContext FEATURE_COLS = ["PetalLength", "PetalWidth", "SepalLength", "SepalWidth"] PREDICTION_COLS = ["value", "confidence"] @@ -104,14 +103,3 @@ async def __aenter__(self) -> "BaseSourceContext": async def __aexit__(self, exc_type, exc_value, traceback): await self.__db.__aexit__(exc_type, exc_value, traceback) - - @classmethod - def args(cls, args, *above) -> Dict[str, Arg]: - cls.config_set(args, above, "filename", Arg()) - return args - - @classmethod - def config(cls, config, *above): - return CustomSQLiteSourceConfig( - filename=cls.config_get(config, above, "filename") - ) diff --git a/examples/source/test_custom_sqlite.py b/examples/source/test_custom_sqlite.py index 630bf05f7b..808a643da2 100644 --- a/examples/source/test_custom_sqlite.py +++ b/examples/source/test_custom_sqlite.py @@ -1,7 +1,6 @@ import unittest -from dffml.util.testing.source import FileSourceTest -from dffml.util.asynctestcase import AsyncTestCase +from dffml import AsyncTestCase, FileSourceTest from .custom_sqlite import CustomSQLiteSourceConfig, CustomSQLiteSource diff --git a/examples/test_quickstart.py b/examples/test_quickstart.py index fb405a8057..fd5a925aee 100644 --- a/examples/test_quickstart.py +++ b/examples/test_quickstart.py @@ -11,8 +11,7 @@ import subprocess import unittest.mock -from dffml.util.os import chdir -from dffml.util.asynctestcase import AsyncTestCase +from dffml import chdir, AsyncTestCase from dffml_service_http.cli import HTTPService from dffml_service_http.util.testing import ServerRunner