github.com/kubeflow/training-operator@v1.7.0/sdk/python/test/e2e/test_e2e_mxjob.py

# Copyright 2021 kubeflow.org.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import logging
import pytest
from typing import Tuple

from kubernetes.client import V1PodTemplateSpec
from kubernetes.client import V1ObjectMeta
from kubernetes.client import V1PodSpec
from kubernetes.client import V1Container
from kubernetes.client import V1ContainerPort
from kubernetes.client import V1ResourceRequirements

from kubeflow.training import TrainingClient
from kubeflow.training import KubeflowOrgV1ReplicaSpec
from kubeflow.training import KubeflowOrgV1MXJob
from kubeflow.training import KubeflowOrgV1MXJobSpec
from kubeflow.training import KubeflowOrgV1RunPolicy
from kubeflow.training import KubeflowOrgV1SchedulingPolicy
from kubeflow.training.constants import constants

from test.e2e.utils import verify_job_e2e, verify_unschedulable_job_e2e, get_pod_spec_scheduler_name
from test.e2e.constants import TEST_GANG_SCHEDULER_NAME_ENV_KEY
from test.e2e.constants import GANG_SCHEDULERS, NONE_GANG_SCHEDULERS

logging.basicConfig(format="%(message)s")
logging.getLogger().setLevel(logging.INFO)

TRAINING_CLIENT = TrainingClient()
JOB_NAME = "mxjob-mnist-ci-test"
CONTAINER_NAME = "mxnet"
GANG_SCHEDULER_NAME = os.getenv(TEST_GANG_SCHEDULER_NAME_ENV_KEY)


@pytest.mark.skipif(
    GANG_SCHEDULER_NAME in NONE_GANG_SCHEDULERS, reason="For gang-scheduling",
)
def test_sdk_e2e_with_gang_scheduling(job_namespace):
    worker_container, server_container, scheduler_container = generate_containers()

    worker = KubeflowOrgV1ReplicaSpec(
        replicas=1,
        restart_policy="Never",
        template=V1PodTemplateSpec(
            metadata=V1ObjectMeta(annotations={constants.ISTIO_SIDECAR_INJECTION: "false"}),
            spec=V1PodSpec(
                containers=[worker_container],
                scheduler_name=get_pod_spec_scheduler_name(GANG_SCHEDULER_NAME),
            )
        ),
    )

    server = KubeflowOrgV1ReplicaSpec(
        replicas=1,
        restart_policy="Never",
        template=V1PodTemplateSpec(
            metadata=V1ObjectMeta(annotations={constants.ISTIO_SIDECAR_INJECTION: "false"}),
            spec=V1PodSpec(
                containers=[server_container],
                scheduler_name=get_pod_spec_scheduler_name(GANG_SCHEDULER_NAME),
            )
        ),
    )

    scheduler = KubeflowOrgV1ReplicaSpec(
        replicas=1,
        restart_policy="Never",
        template=V1PodTemplateSpec(
            metadata=V1ObjectMeta(annotations={constants.ISTIO_SIDECAR_INJECTION: "false"}),
            spec=V1PodSpec(
                containers=[scheduler_container],
                scheduler_name=get_pod_spec_scheduler_name(GANG_SCHEDULER_NAME),
            )
        ),
    )

    unschedulable_mxjob = generate_mxjob(scheduler, server, worker, KubeflowOrgV1SchedulingPolicy(min_available=10), job_namespace)
    schedulable_mxjob = generate_mxjob(scheduler, server, worker, KubeflowOrgV1SchedulingPolicy(min_available=3), job_namespace)

    TRAINING_CLIENT.create_mxjob(unschedulable_mxjob, job_namespace)
    logging.info(f"List of created {constants.MXJOB_KIND}s")
    logging.info(TRAINING_CLIENT.list_mxjobs(job_namespace))

    verify_unschedulable_job_e2e(
        TRAINING_CLIENT,
        JOB_NAME,
        job_namespace,
        constants.MXJOB_KIND,
    )

    TRAINING_CLIENT.patch_mxjob(schedulable_mxjob, JOB_NAME, job_namespace)
    logging.info(f"List of patched {constants.MXJOB_KIND}s")
    logging.info(TRAINING_CLIENT.list_mxjobs(job_namespace))

    verify_job_e2e(
        TRAINING_CLIENT,
        JOB_NAME,
        job_namespace,
        constants.MXJOB_KIND,
        CONTAINER_NAME,
    )

    TRAINING_CLIENT.delete_mxjob(JOB_NAME, job_namespace)


@pytest.mark.skipif(
    GANG_SCHEDULER_NAME in GANG_SCHEDULERS, reason="For plain scheduling",
)
def test_sdk_e2e(job_namespace):
    worker_container, server_container, scheduler_container = generate_containers()

    worker = KubeflowOrgV1ReplicaSpec(
        replicas=1,
        restart_policy="Never",
        template=V1PodTemplateSpec(metadata=V1ObjectMeta(annotations={constants.ISTIO_SIDECAR_INJECTION: "false"}),
                                   spec=V1PodSpec(containers=[worker_container])),
    )

    server = KubeflowOrgV1ReplicaSpec(
        replicas=1,
        restart_policy="Never",
        template=V1PodTemplateSpec(metadata=V1ObjectMeta(annotations={constants.ISTIO_SIDECAR_INJECTION: "false"}),
                                   spec=V1PodSpec(containers=[server_container])),
    )

    scheduler = KubeflowOrgV1ReplicaSpec(
        replicas=1,
        restart_policy="Never",
        template=V1PodTemplateSpec(metadata=V1ObjectMeta(annotations={constants.ISTIO_SIDECAR_INJECTION: "false"}),
                                   spec=V1PodSpec(containers=[scheduler_container])),
    )

    mxjob = generate_mxjob(scheduler, server, worker, job_namespace=job_namespace)

    TRAINING_CLIENT.create_mxjob(mxjob, job_namespace)
    logging.info(f"List of created {constants.MXJOB_KIND}s")
    logging.info(TRAINING_CLIENT.list_mxjobs(job_namespace))

    verify_job_e2e(
        TRAINING_CLIENT,
        JOB_NAME,
        job_namespace,
        constants.MXJOB_KIND,
        CONTAINER_NAME,
    )

    TRAINING_CLIENT.delete_mxjob(JOB_NAME, job_namespace)


def generate_mxjob(
    scheduler: KubeflowOrgV1ReplicaSpec,
    server: KubeflowOrgV1ReplicaSpec,
    worker: KubeflowOrgV1ReplicaSpec,
    scheduling_policy: KubeflowOrgV1SchedulingPolicy = None,
    job_namespace: str = "default",
) -> KubeflowOrgV1MXJob:
    return KubeflowOrgV1MXJob(
        api_version="kubeflow.org/v1",
        kind="MXJob",
        metadata=V1ObjectMeta(name=JOB_NAME, namespace=job_namespace),
        spec=KubeflowOrgV1MXJobSpec(
            job_mode="MXTrain",
            run_policy=KubeflowOrgV1RunPolicy(
                clean_pod_policy="None",
                scheduling_policy=scheduling_policy,
            ),
            mx_replica_specs={
                "Scheduler": scheduler,
                "Server": server,
                "Worker": worker,
            },
        ),
    )


def generate_containers() -> Tuple[V1Container, V1Container, V1Container]:
    worker_container = V1Container(
        name=CONTAINER_NAME,
        # TODO (tenzen-y): Replace the below image with the kubeflow hosted image
        image="docker.io/johnugeorge/mxnet:1.9.1_cpu_py3",
        command=["/usr/local/bin/python3"],
        args=[
            "incubator-mxnet/example/image-classification/train_mnist.py",
            "--num-epochs",
            "1",
            "--num-examples",
            "1000",
            "--kv-store",
            "dist_sync",
        ],
        ports=[V1ContainerPort(container_port=9991, name="mxjob-port")],
        resources=V1ResourceRequirements(limits={"memory": "1Gi", "cpu": "0.25"}),
    )

    server_container = V1Container(
        name=CONTAINER_NAME,
        # TODO (tenzen-y): Replace the below image with the kubeflow hosted image
        image="docker.io/johnugeorge/mxnet:1.9.1_cpu_py3",
        ports=[V1ContainerPort(container_port=9991, name="mxjob-port")],
        resources=V1ResourceRequirements(limits={"memory": "1Gi", "cpu": "0.25"}),
    )

    scheduler_container = V1Container(
        name=CONTAINER_NAME,
        # TODO (tenzen-y): Replace the below image with the kubeflow hosted image
        image="docker.io/johnugeorge/mxnet:1.9.1_cpu_py3",
        ports=[V1ContainerPort(container_port=9991, name="mxjob-port")],
        resources=V1ResourceRequirements(limits={"memory": "1Gi", "cpu": "0.25"}),
    )

    return worker_container, server_container, scheduler_container