Add usage of volume mount example for Kubeflow pipeline. (#1044)

* Add flow control example of Kubeflow pipeline

* Add volume example.

* Upload runs graph.

* Add data passing example of Kubeflow pipeline

* Add sidecar example for Kubeflow pipeline.

pipelines-demo/dsl/data-passing.py

@@ -0,0 +1,57 @@
+#  Copyright 2023 kbthu. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import kfp
+from kfp import dsl
+from kfp.components import func_to_container_op
+from typing import NamedTuple
+
+
+@func_to_container_op
+def calculate_op(a: float, b: float) -> NamedTuple(
+  'MyOutputs',
+  [
+    ('sum', float),
+    ('product', float)
+  ]):
+  return (a + b, a * b)
+
+
+def echo_func(sum: float, product: float):
+  cop = dsl.ContainerOp(
+    name='echo_func',
+    image='bash:5.1',
+    command=['sh', '-c'],
+    arguments=['echo sum %s product %s | tee /out.txt' % (sum, product)],
+	file_outputs={'out': '/out.txt'},
+  )
+  cop.container.set_image_pull_policy('IfNotPresent')
+  return cop
+
+
+@dsl.pipeline(
+  name='Kubeflow data passing example',
+  description='Demonstrate the data passing between Kubeflow funcs'
+)
+def data_passing(
+    a: dsl.PipelineParam(name='a', value=2), 
+    b: dsl.PipelineParam(name='b', value=3)
+  ):
+  result = calculate_op(a, b)
+  echo_func(result.outputs['sum'], result.outputs['product'])
+
+
+if __name__ == '__main__':
+  import kfp.compiler as compiler
+  compiler.Compiler().compile(data_passing, __file__ + '.yaml')

pipelines-demo/dsl/sidecar-example.py

@@ -0,0 +1,32 @@
+#  Copyright 2023 kbthu. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import kfp
+from kfp import dsl
+
+@dsl.pipeline(
+    name='metric-test',
+    description='query metrics from promehteus')
+def exec_pipeline():
+  op = dsl.ContainerOp(name='metrics-test',
+                      image='kbthu/metrics-example:1.0.1',
+                      command=['metrics'],
+                      arguments=['pq', '-q', 'count(prometheus_target_interval_length_seconds)'],
+  )
+  op.container.set_image_pull_policy('IfNotPresent')
+  op.add_sidecar(dsl.Sidecar('prometheus', 'bitnami/prometheus:2.44.0', command='ls')
+    .set_image_pull_policy('IfNotPresent'))
+
+if __name__ == '__main__':
+    kfp.compiler.Compiler().compile(exec_pipeline, __file__ + '.yaml')

pipelines-demo/volume/volume_example.py

@@ -0,0 +1,63 @@
+#  Copyright 2023 kbthu. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import kfp
+from kfp import dsl
+
+def create_pv():
+  return dsl.VolumeOp(
+    name="create_pv",
+    resource_name="kfp-pvc",
+    size="1Gi",
+    modes=dsl.VOLUME_MODE_RWO
+  )
+
+
+def generate_data(vol_name: str):
+  cop = dsl.ContainerOp(
+    name='generate_data',
+    image='bash:5.1',
+    command=['sh', '-c'],
+    arguments=['echo $(( $RANDOM % 10 + 1 )) | tee /mnt/out.txt']
+  )
+  cop.container.set_image_pull_policy('IfNotPresent')
+  cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)})
+  return cop
+
+
+def use_pre_data(vol_name: str):
+  cop = dsl.ContainerOp(
+    name='use_pre_data',
+    image='bash:5.1',
+    command=['sh', '-c'],
+    arguments=['tail /mnt/out.txt']
+  )
+  cop.container.set_image_pull_policy('IfNotPresent')
+  cop.add_pvolumes({'/mnt': dsl.PipelineVolume(pvc=vol_name)})
+  return cop
+
+
+@dsl.pipeline(
+    name="Kubeflow volume example",
+    description="Demonstrate the use case of volume on Kubeflow pipeline."
+)
+def volume_example():
+  vop = create_pv()
+  cop = generate_data(vop.outputs["name"]).after(vop)
+  use_pre_data(vop.outputs["name"]).after(vop,cop)
+
+
+if __name__ == "__main__":
+    import kfp.compiler as compiler
+    compiler.Compiler().compile(volume_example, __file__ + ".yaml")