335 lines
14 KiB
YAML
335 lines
14 KiB
YAML
# PIPELINE DEFINITION
|
||
# Name: cats-vs-dogs-classification-from-minio
|
||
# Description: A pipeline to train and evaluate a cats-vs-dogs classifier using dataset from MinIO.
|
||
# Inputs:
|
||
# batch_size: int [Default: 32.0]
|
||
# bucket_name: str [Default: 'xrwang']
|
||
# epochs: int [Default: 10.0]
|
||
# local_zip_path: str [Default: '/tmp/cat-dog_data.zip']
|
||
# minio_access_key: str [Default: 'LEINAOYUNOS']
|
||
# minio_endpoint: str [Default: 'miniogw-dev2.cnbita.com:11443']
|
||
# minio_secret_key: str [Default: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYLEINAOYUNKEY']
|
||
# object_name: str [Default: 'catdog/cat-dog_data.zip']
|
||
# secure: bool [Default: True]
|
||
# Outputs:
|
||
# evaluate-model-metrics: system.Metrics
|
||
|
||
components:
|
||
#组件的输入输出类型
|
||
comp-download-and-prepare-data-from-minio:
|
||
executorLabel: exec-download-and-prepare-data-from-minio
|
||
inputDefinitions:
|
||
parameters:
|
||
bucket_name:
|
||
parameterType: STRING
|
||
local_zip_path:
|
||
parameterType: STRING
|
||
minio_access_key:
|
||
parameterType: STRING
|
||
minio_endpoint:
|
||
parameterType: STRING
|
||
minio_secret_key:
|
||
parameterType: STRING
|
||
object_name:
|
||
parameterType: STRING
|
||
secure:
|
||
defaultValue: true
|
||
isOptional: true
|
||
parameterType: BOOLEAN
|
||
outputDefinitions:
|
||
artifacts:
|
||
dataset_output:
|
||
artifactType:
|
||
schemaTitle: system.Dataset
|
||
schemaVersion: 0.0.1
|
||
comp-evaluate-model:
|
||
executorLabel: exec-evaluate-model
|
||
inputDefinitions:
|
||
artifacts:
|
||
dataset_input:
|
||
artifactType:
|
||
schemaTitle: system.Dataset
|
||
schemaVersion: 0.0.1
|
||
model_dir:
|
||
artifactType:
|
||
schemaTitle: system.Model
|
||
schemaVersion: 0.0.1
|
||
outputDefinitions:
|
||
artifacts:
|
||
metrics:
|
||
artifactType:
|
||
schemaTitle: system.Metrics
|
||
schemaVersion: 0.0.1
|
||
comp-train-model:
|
||
executorLabel: exec-train-model
|
||
inputDefinitions:
|
||
artifacts:
|
||
dataset_input:
|
||
artifactType:
|
||
schemaTitle: system.Dataset
|
||
schemaVersion: 0.0.1
|
||
parameters:
|
||
batch_size:
|
||
parameterType: NUMBER_INTEGER
|
||
epochs:
|
||
parameterType: NUMBER_INTEGER
|
||
outputDefinitions:
|
||
artifacts:
|
||
model_output:
|
||
artifactType:
|
||
schemaTitle: system.Model
|
||
schemaVersion: 0.0.1
|
||
defaultPipelineRoot: minio://xrwang/artifacts/
|
||
deploymentSpec:
|
||
executors:
|
||
exec-download-and-prepare-data-from-minio:
|
||
container:
|
||
args:
|
||
- --executor_input
|
||
- "{{$}}"
|
||
- --function_to_execute
|
||
- download_and_prepare_data_from_minio
|
||
command:
|
||
- sh
|
||
- -c
|
||
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
|
||
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
|
||
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\
|
||
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' &&\
|
||
\ python3 -m pip install --quiet --no-warn-script-location 'minio' 'pandas'\
|
||
\ 'tensorflow' 'pathlib' 'zipfile36' && \"$0\" \"$@\"\n"
|
||
- sh
|
||
- -ec
|
||
- 'program_path=$(mktemp -d)
|
||
|
||
|
||
printf "%s" "$0" > "$program_path/ephemeral_component.py"
|
||
|
||
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
|
||
|
||
'
|
||
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
|
||
\ *\n\ndef download_and_prepare_data_from_minio(\n dataset_output: Output[Dataset],\n\
|
||
\ minio_endpoint: str,\n minio_access_key: str,\n minio_secret_key:\
|
||
\ str,\n bucket_name: str,\n object_name: str,\n local_zip_path:\
|
||
\ str,\n secure: bool = True\n):\n from minio import Minio\n from\
|
||
\ zipfile import ZipFile\n from pathlib import Path\n import os\n\
|
||
\ client = Minio(\n minio_endpoint,\n access_key=minio_access_key,\n\
|
||
\ secret_key=minio_secret_key,\n secure=secure\n )\n \
|
||
\ print (\"download_and_prepare_data_from_minio\")\n client.fget_object(bucket_name,\
|
||
\ object_name, local_zip_path)\n with ZipFile(local_zip_path, 'r') as\
|
||
\ zip_ref:\n zip_ref.extractall(path=dataset_output.path)\n os.remove(local_zip_path)\n\
|
||
\ print(f\"Data prepared at: {dataset_output.path}\")\n\n"
|
||
image: python:3.7
|
||
exec-evaluate-model:
|
||
container:
|
||
args:
|
||
- --executor_input
|
||
- "{{$}}"
|
||
- --function_to_execute
|
||
- evaluate_model
|
||
command:
|
||
- sh
|
||
- -c
|
||
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
|
||
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
|
||
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\
|
||
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' &&\
|
||
\ python3 -m pip install --quiet --no-warn-script-location 'tensorflow'\
|
||
\ 'pandas' && \"$0\" \"$@\"\n"
|
||
- sh
|
||
- -ec
|
||
- 'program_path=$(mktemp -d)
|
||
|
||
|
||
printf "%s" "$0" > "$program_path/ephemeral_component.py"
|
||
|
||
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
|
||
|
||
'
|
||
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
|
||
\ *\n\ndef evaluate_model(\n model_dir: Input[Model],\n dataset_input:\
|
||
\ Input[Dataset],\n metrics: Output[Metrics]\n):\n import tensorflow\
|
||
\ as tf\n import os\n\n # \u52A0\u8F7D\u6A21\u578B\n model = tf.keras.models.load_model(model_dir.path)\n\
|
||
\n # \u52A0\u8F7D\u9A8C\u8BC1\u6570\u636E\u96C6\n validation_dir =\
|
||
\ os.path.join(dataset_input.path, 'validation')\n validation_dataset\
|
||
\ = tf.keras.utils.image_dataset_from_directory(\n validation_dir,\n\
|
||
\ image_size=(180, 180),\n batch_size=32)\n\n # \u8BC4\u4F30\
|
||
\u6A21\u578B\n eval_result = model.evaluate(validation_dataset)\n\n \
|
||
\ # \u8BB0\u5F55\u8BC4\u4F30\u7ED3\u679C\n metrics.log_metric(\"loss\"\
|
||
, eval_result[0])\n metrics.log_metric(\"accuracy\", eval_result[1])\n\
|
||
\n"
|
||
image: registry.cnbita.com:5000/kubeflow-pipelines/python-tensorflow:3.7
|
||
exec-train-model:
|
||
container:
|
||
args:
|
||
- --executor_input
|
||
- "{{$}}"
|
||
- --function_to_execute
|
||
- train_model
|
||
command:
|
||
- sh
|
||
- -c
|
||
- "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\
|
||
\ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
|
||
\ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.7.0'\
|
||
\ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' &&\
|
||
\ python3 -m pip install --quiet --no-warn-script-location 'tensorflow'\
|
||
\ 'numpy' && \"$0\" \"$@\"\n"
|
||
- sh
|
||
- -ec
|
||
- 'program_path=$(mktemp -d)
|
||
|
||
|
||
printf "%s" "$0" > "$program_path/ephemeral_component.py"
|
||
|
||
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
|
||
|
||
'
|
||
- "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
|
||
\ *\n\ndef train_model(\n dataset_input: Input[Dataset],\n model_output:\
|
||
\ Output[Model],\n epochs: int,\n batch_size: int\n):\n import\
|
||
\ tensorflow as tf\n import os\n train_dir = os.path.join(dataset_input.path,\
|
||
\ 'train')\n validation_dir = os.path.join(dataset_input.path, 'validation')\n\
|
||
\n train_dataset = tf.keras.utils.image_dataset_from_directory(\n \
|
||
\ train_dir,\n image_size=(180, 180),\n batch_size=batch_size)\n\
|
||
\n validation_dataset = tf.keras.utils.image_dataset_from_directory(\n\
|
||
\ validation_dir,\n image_size=(180, 180),\n batch_size=batch_size)\n\
|
||
\n model = tf.keras.Sequential([\n tf.keras.layers.Rescaling(1./255),\n\
|
||
\ tf.keras.layers.Conv2D(16, 3, padding='same', activation='relu'),\n\
|
||
\ tf.keras.layers.MaxPooling2D(),\n tf.keras.layers.Conv2D(32,\
|
||
\ 3, padding='same', activation='relu'),\n tf.keras.layers.MaxPooling2D(),\n\
|
||
\ tf.keras.layers.Conv2D(64, 3, padding='same', activation='relu'),\n\
|
||
\ tf.keras.layers.MaxPooling2D(),\n tf.keras.layers.Flatten(),\n\
|
||
\ tf.keras.layers.Dense(128, activation='relu'), \n tf.keras.layers.Dense(2)\n\
|
||
\n ])\n\n model.compile(optimizer='adam',\n loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n\
|
||
\ metrics=['accuracy'])\n\n model.fit(train_dataset,\
|
||
\ validation_data=validation_dataset, epochs=epochs, batch_size=batch_size)\n\
|
||
\ model.save(model_output.path)\n\n"
|
||
image: registry.cnbita.com:5000/kubeflow-pipelines/python-tensorflow:3.7
|
||
pipelineInfo:
|
||
description:
|
||
A pipeline to train and evaluate a cats-vs-dogs classifier using dataset
|
||
from MinIO.
|
||
name: cats-vs-dogs-classification-from-minio
|
||
root:
|
||
dag:
|
||
outputs:
|
||
artifacts:
|
||
evaluate-model-metrics:
|
||
artifactSelectors:
|
||
- outputArtifactKey: metrics
|
||
producerSubtask: evaluate-model
|
||
#·、该组件的输入参数来源哪个组件 2、组件之间的依赖 3、组件的名称（唯一标识）
|
||
tasks:
|
||
download-and-prepare-data-from-minio:
|
||
cachingOptions:
|
||
enableCache: true
|
||
componentRef:
|
||
name: comp-download-and-prepare-data-from-minio
|
||
inputs:
|
||
parameters:
|
||
bucket_name:
|
||
componentInputParameter: bucket_name
|
||
local_zip_path:
|
||
componentInputParameter: local_zip_path
|
||
minio_access_key:
|
||
componentInputParameter: minio_access_key
|
||
minio_endpoint:
|
||
componentInputParameter: minio_endpoint
|
||
minio_secret_key:
|
||
componentInputParameter: minio_secret_key
|
||
object_name:
|
||
componentInputParameter: object_name
|
||
secure:
|
||
componentInputParameter: secure
|
||
taskInfo:
|
||
name: download-and-prepare-data-from-minio
|
||
evaluate-model:
|
||
cachingOptions: {}
|
||
#当前组件（evaluate-model）的定义来源（comp-evaluate-model）
|
||
componentRef:
|
||
name: comp-evaluate-model
|
||
#放在哪些组件之后跑
|
||
dependentTasks:
|
||
- download-and-prepare-data-from-minio
|
||
- train-model
|
||
# 该组件的输入参数来源哪个组件
|
||
inputs:
|
||
artifacts:
|
||
dataset_input:
|
||
taskOutputArtifact:
|
||
outputArtifactKey: dataset_output
|
||
producerTask: download-and-prepare-data-from-minio
|
||
model_dir:
|
||
taskOutputArtifact:
|
||
outputArtifactKey: model_output
|
||
producerTask: train-model
|
||
taskInfo:
|
||
name: evaluate-model
|
||
train-model:
|
||
cachingOptions: {}
|
||
componentRef:
|
||
name: comp-train-model
|
||
dependentTasks:
|
||
- download-and-prepare-data-from-minio
|
||
inputs:
|
||
artifacts:
|
||
dataset_input:
|
||
taskOutputArtifact:
|
||
outputArtifactKey: dataset_output
|
||
producerTask: download-and-prepare-data-from-minio
|
||
parameters:
|
||
batch_size:
|
||
componentInputParameter: batch_size
|
||
epochs:
|
||
componentInputParameter: epochs
|
||
taskInfo:
|
||
name: train-model
|
||
inputDefinitions:
|
||
parameters:
|
||
batch_size:
|
||
defaultValue: 32.0
|
||
isOptional: true
|
||
parameterType: NUMBER_INTEGER
|
||
bucket_name:
|
||
defaultValue: xrwang
|
||
isOptional: true
|
||
parameterType: STRING
|
||
epochs:
|
||
defaultValue: 10.0
|
||
isOptional: true
|
||
parameterType: NUMBER_INTEGER
|
||
local_zip_path:
|
||
defaultValue: /tmp/cat-dog_data.zip
|
||
isOptional: true
|
||
parameterType: STRING
|
||
minio_access_key:
|
||
defaultValue: LEINAOYUNOS
|
||
isOptional: true
|
||
parameterType: STRING
|
||
minio_endpoint:
|
||
defaultValue: miniogw-dev2.cnbita.com:11443
|
||
isOptional: true
|
||
parameterType: STRING
|
||
minio_secret_key:
|
||
defaultValue: wJalrXUtnFEMI/K7MDENG/bPxRfiCYLEINAOYUNKEY
|
||
isOptional: true
|
||
parameterType: STRING
|
||
object_name:
|
||
defaultValue: catdog/cat-dog_data.zip
|
||
isOptional: true
|
||
parameterType: STRING
|
||
secure:
|
||
defaultValue: true
|
||
isOptional: true
|
||
parameterType: BOOLEAN
|
||
outputDefinitions:
|
||
artifacts:
|
||
evaluate-model-metrics:
|
||
artifactType:
|
||
schemaTitle: system.Metrics
|
||
schemaVersion: 0.0.1
|
||
schemaVersion: 2.1.0
|
||
sdkVersion: kfp-2.7.0
|