Merge pull request GoogleCloudPlatform#173 from GoogleCloudPlatform/k…

…fp-comp-update-issue-170 Updated create_component_from_func_v2 to @component decorator
ozagordi · Mar 9, 2022 · 29bfbe9 · 29bfbe9
2 parents e87f351 + 9c56a38
commit 29bfbe9
Show file tree

Hide file tree

Showing 14 changed files with 63 additions and 92 deletions.
diff --git a/notebooks/kubeflow_pipelines/cicd/labs/pipeline_vertex/pipeline.py b/notebooks/kubeflow_pipelines/cicd/labs/pipeline_vertex/pipeline.py
@@ -15,7 +15,6 @@
 import os
 
 from kfp import dsl
-from kfp.components import create_component_from_func_v2
 from training_lightweight_component import train_and_deploy
 from tuning_lightweight_component import tune_hyperparameters
 
@@ -34,22 +33,6 @@
 THRESHOLD = float(os.getenv("THRESHOLD", "0.6"))
 
 
-tune_hyperparameters_component = create_component_from_func_v2(
-    tune_hyperparameters,
-    base_image="python:3.8",
-    output_component_file="covertype_kfp_tune_hyperparameters.yaml",
-    packages_to_install=["google-cloud-aiplatform"],
-)
-
-
-train_and_deploy_component = create_component_from_func_v2(
-    train_and_deploy,
-    base_image="python:3.8",
-    output_component_file="covertype_kfp_train_and_deploy.yaml",
-    packages_to_install=["google-cloud-aiplatform"],
-)
-
-
 @dsl.pipeline(
     name="covertype-kfp-pipeline",
     description="The pipeline training and deploying the Covertype classifier",
@@ -67,7 +50,7 @@ def covertype_train(
 ):
     staging_bucket = f"{pipeline_root}/staging"
 
-    tuning_op = tune_hyperparameters_component(
+    tuning_op = tune_hyperparameters(
         project=PROJECT_ID,
         location=REGION,
         container_uri=training_container_uri,
@@ -84,7 +67,7 @@ def covertype_train(
         accuracy >= accuracy_deployment_threshold, name="deploy_decision"
     ):
         train_and_deploy_op = (  # pylint: disable=unused-variable
-            train_and_deploy_component(
+            train_and_deploy(
                 project=PROJECT_ID,
                 location=REGION,
                 container_uri=training_container_uri,

diff --git a/notebooks/kubeflow_pipelines/cicd/labs/pipeline_vertex/training_lightweight_component.py b/notebooks/kubeflow_pipelines/cicd/labs/pipeline_vertex/training_lightweight_component.py
@@ -12,8 +12,14 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 """Lightweight component training function."""
+from kfp.v2.dsl import component
 
 
+@component(
+    base_image="python:3.8",
+    output_component_file="covertype_kfp_train_and_deploy.yaml",
+    packages_to_install=["google-cloud-aiplatform"],
+)
 def train_and_deploy(
     project: str,
     location: str,

diff --git a/notebooks/kubeflow_pipelines/cicd/labs/pipeline_vertex/tuning_lightweight_component.py b/notebooks/kubeflow_pipelines/cicd/labs/pipeline_vertex/tuning_lightweight_component.py
@@ -14,7 +14,14 @@
 """Lightweight component tuning function."""
 from typing import NamedTuple
 
+from kfp.v2.dsl import component
 
+
+@component(
+    base_image="python:3.8",
+    output_component_file="covertype_kfp_tune_hyperparameters.yaml",
+    packages_to_install=["google-cloud-aiplatform"],
+)
 def tune_hyperparameters(
     project: str,
     location: str,

diff --git a/notebooks/kubeflow_pipelines/cicd/solutions/pipeline_vertex/pipeline.py b/notebooks/kubeflow_pipelines/cicd/solutions/pipeline_vertex/pipeline.py
@@ -15,7 +15,6 @@
 import os
 
 from kfp import dsl
-from kfp.components import create_component_from_func_v2
 from training_lightweight_component import train_and_deploy
 from tuning_lightweight_component import tune_hyperparameters
 
@@ -34,22 +33,6 @@
 THRESHOLD = float(os.getenv("THRESHOLD", "0.6"))
 
 
-tune_hyperparameters_component = create_component_from_func_v2(
-    tune_hyperparameters,
-    base_image="python:3.8",
-    output_component_file="covertype_kfp_tune_hyperparameters.yaml",
-    packages_to_install=["google-cloud-aiplatform"],
-)
-
-
-train_and_deploy_component = create_component_from_func_v2(
-    train_and_deploy,
-    base_image="python:3.8",
-    output_component_file="covertype_kfp_train_and_deploy.yaml",
-    packages_to_install=["google-cloud-aiplatform"],
-)
-
-
 @dsl.pipeline(
     name="covertype-kfp-pipeline",
     description="The pipeline training and deploying the Covertype classifier",
@@ -67,7 +50,7 @@ def covertype_train(
 ):
     staging_bucket = f"{pipeline_root}/staging"
 
-    tuning_op = tune_hyperparameters_component(
+    tuning_op = tune_hyperparameters(
         project=PROJECT_ID,
         location=REGION,
         container_uri=training_container_uri,
@@ -84,7 +67,7 @@ def covertype_train(
         accuracy >= accuracy_deployment_threshold, name="deploy_decision"
     ):
         train_and_deploy_op = (  # pylint: disable=unused-variable
-            train_and_deploy_component(
+            train_and_deploy(
                 project=PROJECT_ID,
                 location=REGION,
                 container_uri=training_container_uri,

diff --git a/...books/kubeflow_pipelines/cicd/solutions/pipeline_vertex/training_lightweight_component.py b/...books/kubeflow_pipelines/cicd/solutions/pipeline_vertex/training_lightweight_component.py
@@ -12,8 +12,14 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 """Lightweight component training function."""
+from kfp.v2.dsl import component
 
 
+@component(
+    base_image="python:3.8",
+    output_component_file="covertype_kfp_train_and_deploy.yaml",
+    packages_to_install=["google-cloud-aiplatform"],
+)
 def train_and_deploy(
     project: str,
     location: str,

diff --git a/notebooks/kubeflow_pipelines/cicd/solutions/pipeline_vertex/tuning_lightweight_component.py b/notebooks/kubeflow_pipelines/cicd/solutions/pipeline_vertex/tuning_lightweight_component.py
@@ -14,7 +14,14 @@
 """Lightweight component tuning function."""
 from typing import NamedTuple
 
+from kfp.v2.dsl import component
 
+
+@component(
+    base_image="python:3.8",
+    output_component_file="covertype_kfp_tune_hyperparameters.yaml",
+    packages_to_install=["google-cloud-aiplatform"],
+)
 def tune_hyperparameters(
     project: str,
     location: str,

diff --git a/notebooks/kubeflow_pipelines/pipelines/labs/kfp_pipeline_vertex.ipynb b/notebooks/kubeflow_pipelines/pipelines/labs/kfp_pipeline_vertex.ipynb
@@ -204,7 +204,6 @@
     "import os\n",
     "\n",
     "from kfp import dsl\n",
-    "from kfp.components import create_component_from_func_v2\n",
     "from training_lightweight_component import train_and_deploy\n",
     "from tuning_lightweight_component import tune_hyperparameters\n",
     "\n",
@@ -218,15 +217,9 @@
     "TRAINING_FILE_PATH = os.getenv(\"TRAINING_FILE_PATH\")\n",
     "VALIDATION_FILE_PATH = os.getenv(\"VALIDATION_FILE_PATH\")\n",
     "\n",
-    "MAX_TRIAL_COUNT = os.getenv(\"MAX_TRIAL_COUNT\", 5)\n",
-    "PARALLEL_TRIAL_COUNT = os.getenv(\"PARALLEL_TRIAL_COUNT\", 5)\n",
-    "THRESHOLD = os.getenv(\"THRESHOLD\", 0.6)\n",
-    "\n",
-    "\n",
-    "tune_hyperparameters_component =  # TODO\n",
-    "\n",
-    "\n",
-    "train_and_deploy_component =  # TODO\n",
+    "MAX_TRIAL_COUNT = int(os.getenv(\"MAX_TRIAL_COUNT\", \"5\"))\n",
+    "PARALLEL_TRIAL_COUNT = int(os.getenv(\"PARALLEL_TRIAL_COUNT\", \"5\"))\n",
+    "THRESHOLD = float(os.getenv(\"THRESHOLD\", \"0.6\"))\n",
     "\n",
     "\n",
     "@dsl.pipeline(\n",

diff --git a/notebooks/kubeflow_pipelines/pipelines/labs/pipeline_vertex/pipeline.py b/notebooks/kubeflow_pipelines/pipelines/labs/pipeline_vertex/pipeline.py
@@ -16,7 +16,6 @@
 import os
 
 from kfp import dsl
-from kfp.components import create_component_from_func_v2
 from training_lightweight_component import train_and_deploy
 from tuning_lightweight_component import tune_hyperparameters
 
@@ -35,11 +34,6 @@
 THRESHOLD = float(os.getenv("THRESHOLD", "0.6"))
 
 
-tune_hyperparameters_component = None  # TODO
-
-train_and_deploy_component = None  # TODO
-
-
 @dsl.pipeline(
     name="covertype-kfp-pipeline",
     description="The pipeline training and deploying the Covertype classifier",

diff --git a/...books/kubeflow_pipelines/pipelines/labs/pipeline_vertex/training_lightweight_component.py b/...books/kubeflow_pipelines/pipelines/labs/pipeline_vertex/training_lightweight_component.py
@@ -12,9 +12,15 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 """Lightweight component training function."""
+from kfp.v2.dsl import component
 
 
 # pylint: disable=unused-argument
+@component(
+    base_image="python:3.8",
+    output_component_file="covertype_kfp_train_and_deploy.yaml",
+    packages_to_install=["google-cloud-aiplatform"],
+)
 def train_and_deploy(
     project: str,
     location: str,

diff --git a/notebooks/kubeflow_pipelines/pipelines/labs/pipeline_vertex/tuning_lightweight_component.py b/notebooks/kubeflow_pipelines/pipelines/labs/pipeline_vertex/tuning_lightweight_component.py
@@ -14,8 +14,15 @@
 """Lightweight component tuning function."""
 from typing import NamedTuple
 
+from kfp.v2.dsl import component
+
 
 # pylint: disable=unused-argument
+@component(
+    base_image="python:3.8",
+    output_component_file="covertype_kfp_tune_hyperparameters.yaml",
+    packages_to_install=["google-cloud-aiplatform"],
+)
 def tune_hyperparameters(
     project: str,
     location: str,

diff --git a/notebooks/kubeflow_pipelines/pipelines/solutions/kfp_pipeline_vertex.ipynb b/notebooks/kubeflow_pipelines/pipelines/solutions/kfp_pipeline_vertex.ipynb
@@ -193,7 +193,6 @@
     "import os\n",
     "\n",
     "from kfp import dsl\n",
-    "from kfp.components import create_component_from_func_v2\n",
     "from training_lightweight_component import train_and_deploy\n",
     "from tuning_lightweight_component import tune_hyperparameters\n",
     "\n",
@@ -212,22 +211,6 @@
     "THRESHOLD = float(os.getenv(\"THRESHOLD\", \"0.6\"))\n",
     "\n",
     "\n",
-    "tune_hyperparameters_component = create_component_from_func_v2(\n",
-    "    tune_hyperparameters,\n",
-    "    base_image=\"python:3.8\",\n",
-    "    output_component_file=\"covertype_kfp_tune_hyperparameters.yaml\",\n",
-    "    packages_to_install=[\"google-cloud-aiplatform\"],\n",
-    ")\n",
-    "\n",
-    "\n",
-    "train_and_deploy_component = create_component_from_func_v2(\n",
-    "    train_and_deploy,\n",
-    "    base_image=\"python:3.8\",\n",
-    "    output_component_file=\"covertype_kfp_train_and_deploy.yaml\",\n",
-    "    packages_to_install=[\"google-cloud-aiplatform\"],\n",
-    ")\n",
-    "\n",
-    "\n",
     "@dsl.pipeline(\n",
     "    name=\"covertype-kfp-pipeline\",\n",
     "    description=\"The pipeline training and deploying the Covertype classifier\",\n",
@@ -245,7 +228,7 @@
     "):\n",
     "    staging_bucket = f\"{pipeline_root}/staging\"\n",
     "\n",
-    "    tuning_op = tune_hyperparameters_component(\n",
+    "    tuning_op = tune_hyperparameters(\n",
     "        project=PROJECT_ID,\n",
     "        location=REGION,\n",
     "        container_uri=training_container_uri,\n",
@@ -262,7 +245,7 @@
     "        accuracy >= accuracy_deployment_threshold, name=\"deploy_decision\"\n",
     "    ):\n",
     "        train_and_deploy_op = (  # pylint: disable=unused-variable\n",
-    "            train_and_deploy_component(\n",
+    "            train_and_deploy(\n",
     "                project=PROJECT_ID,\n",
     "                location=REGION,\n",
     "                container_uri=training_container_uri,\n",

diff --git a/notebooks/kubeflow_pipelines/pipelines/solutions/pipeline_vertex/pipeline.py b/notebooks/kubeflow_pipelines/pipelines/solutions/pipeline_vertex/pipeline.py
@@ -15,7 +15,6 @@
 import os
 
 from kfp import dsl
-from kfp.components import create_component_from_func_v2
 from training_lightweight_component import train_and_deploy
 from tuning_lightweight_component import tune_hyperparameters
 
@@ -34,22 +33,6 @@
 THRESHOLD = float(os.getenv("THRESHOLD", "0.6"))
 
 
-tune_hyperparameters_component = create_component_from_func_v2(
-    tune_hyperparameters,
-    base_image="python:3.8",
-    output_component_file="covertype_kfp_tune_hyperparameters.yaml",
-    packages_to_install=["google-cloud-aiplatform"],
-)
-
-
-train_and_deploy_component = create_component_from_func_v2(
-    train_and_deploy,
-    base_image="python:3.8",
-    output_component_file="covertype_kfp_train_and_deploy.yaml",
-    packages_to_install=["google-cloud-aiplatform"],
-)
-
-
 @dsl.pipeline(
     name="covertype-kfp-pipeline",
     description="The pipeline training and deploying the Covertype classifier",
@@ -67,7 +50,7 @@ def covertype_train(
 ):
     staging_bucket = f"{pipeline_root}/staging"
 
-    tuning_op = tune_hyperparameters_component(
+    tuning_op = tune_hyperparameters(
         project=PROJECT_ID,
         location=REGION,
         container_uri=training_container_uri,
@@ -84,7 +67,7 @@ def covertype_train(
         accuracy >= accuracy_deployment_threshold, name="deploy_decision"
     ):
         train_and_deploy_op = (  # pylint: disable=unused-variable
-            train_and_deploy_component(
+            train_and_deploy(
                 project=PROJECT_ID,
                 location=REGION,
                 container_uri=training_container_uri,

diff --git a/.../kubeflow_pipelines/pipelines/solutions/pipeline_vertex/training_lightweight_component.py b/.../kubeflow_pipelines/pipelines/solutions/pipeline_vertex/training_lightweight_component.py
@@ -12,8 +12,14 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 """Lightweight component training function."""
+from kfp.v2.dsl import component
 
 
+@component(
+    base_image="python:3.8",
+    output_component_file="covertype_kfp_train_and_deploy.yaml",
+    packages_to_install=["google-cloud-aiplatform"],
+)
 def train_and_deploy(
     project: str,
     location: str,

diff --git a/...ks/kubeflow_pipelines/pipelines/solutions/pipeline_vertex/tuning_lightweight_component.py b/...ks/kubeflow_pipelines/pipelines/solutions/pipeline_vertex/tuning_lightweight_component.py
@@ -14,7 +14,14 @@
 """Lightweight component tuning function."""
 from typing import NamedTuple
 
+from kfp.v2.dsl import component
 
+
+@component(
+    base_image="python:3.8",
+    output_component_file="covertype_kfp_tune_hyperparameters.yaml",
+    packages_to_install=["google-cloud-aiplatform"],
+)
 def tune_hyperparameters(
     project: str,
     location: str,