Working with distributed workloads

# Install PyTorch
RUN python3 -m pip install torch==2.4.0

$ pip install kfp

from kfp import dsl


@dsl.component(
    base_image="registry.redhat.io/ubi8/python-39:latest",
    packages_to_install=['codeflare-sdk']
)


def ray_fn():
   import ray (1)
   from codeflare_sdk import Cluster, ClusterConfiguration, generate_cert (2)


   cluster = Cluster( (3)
       ClusterConfiguration(
           namespace="my_project", (4)
           name="raytest",
           num_workers=1,
           head_cpus="500m",
           min_memory=1,
           max_memory=1,
           worker_extended_resource_requests={“nvidia.com/gpu”: 1}, (5)
           image="quay.io/modh/ray:2.35.0-py39-cu121", (6)
           local_queue="local_queue_name", (7)
       )
   )


   print(cluster.status())
   cluster.up() (8)
   cluster.wait_ready() (9)
   print(cluster.status())
   print(cluster.details())


   ray_dashboard_uri = cluster.cluster_dashboard_uri()
   ray_cluster_uri = cluster.cluster_uri()
   print(ray_dashboard_uri, ray_cluster_uri)

   # Enable Ray client to connect to secure Ray cluster that has mTLS enabled
   generate_cert.generate_tls_cert(cluster.config.name, cluster.config.namespace) (10)
   generate_cert.export_env(cluster.config.name, cluster.config.namespace)


   ray.init(address=ray_cluster_uri)
   print("Ray cluster is up and running: ", ray.is_initialized())


   @ray.remote
   def train_fn(): (11)
       # complex training function
       return 100


   result = ray.get(train_fn.remote())
   assert 100 == result
   ray.shutdown()
   cluster.down() (12)
   auth.logout()
   return result


@dsl.pipeline( (13)
   name="Ray Simple Example",
   description="Ray Simple Example",
)


def ray_integration():
   ray_fn()


if __name__ == '__main__': (14)
    from kfp.compiler import Compiler
    Compiler().compile(ray_integration, 'compiled-example.yaml')

$ python compile_example.py

kind: ConfigMap
apiVersion: v1
metadata:
  name: training-config
  namespace: kfto
data:
  config.json: |
    {
      "model_name_or_path": "bigscience/bloom-560m",
      "training_data_path": "/data/input/twitter_complaints.json",
      "output_dir": "/data/output/tuning/bloom-twitter",
      "save_model_dir": "/mnt/output/model",
      "num_train_epochs": 10.0,
      "per_device_train_batch_size": 4,
      "per_device_eval_batch_size": 4,
      "gradient_accumulation_steps": 4,
      "save_strategy": "no",
      "learning_rate": 1e-05,
      "weight_decay": 0.0,
      "lr_scheduler_type": "cosine",
      "include_tokens_per_second": true,
      "response_template": "\n### Label:",
      "dataset_text_field": "output",
      "padding_free": ["huggingface"],
      "multipack": [16],
      "use_flash_attn": false
    }

$ oc apply -f config_trainingjob.yaml

kind: ConfigMap
apiVersion: v1
metadata:
  name: twitter-complaints
  namespace: kfto
data:
  twitter_complaints.json: |
    [
        {"Tweet text":"@HMRCcustomers No this is my first job","ID":0,"Label":2,"text_label":"no complaint","output":"### Text: @HMRCcustomers No this is my first job\n\n### Label: no complaint"},
        {"Tweet text":"@KristaMariePark Thank you for your interest! If you decide to cancel, you can call Customer Care at 1-800-NYTIMES.","ID":1,"Label":2,"text_label":"no complaint","output":"### Text: @KristaMariePark Thank you for your interest! If you decide to cancel, you can call Customer Care at 1-800-NYTIMES.\n\n### Label: no complaint"},
        {"Tweet text":"@EE On Rosneath Arial having good upload and download speeds but terrible latency 200ms. Why is this.","ID":3,"Label":1,"text_label":"complaint","output":"### Text: @EE On Rosneath Arial having good upload and download speeds but terrible latency 200ms. Why is this.\n\n### Label: complaint"},
        {"Tweet text":"Couples wallpaper, so cute. :) #BrothersAtHome","ID":4,"Label":2,"text_label":"no complaint","output":"### Text: Couples wallpaper, so cute. :) #BrothersAtHome\n\n### Label: no complaint"},
        {"Tweet text":"@mckelldogs This might just be me, but-- eyedrops? Artificial tears are so useful when you're sleep-deprived and sp… https:\/\/t.co\/WRtNsokblG","ID":5,"Label":2,"text_label":"no complaint","output":"### Text: @mckelldogs This might just be me, but-- eyedrops? Artificial tears are so useful when you're sleep-deprived and sp… https:\/\/t.co\/WRtNsokblG\n\n### Label: no complaint"},
        {"Tweet text":"@Yelp can we get the exact calculations for a business rating (for example if its 4 stars but actually 4.2) or do we use a 3rd party site?","ID":6,"Label":2,"text_label":"no complaint","output":"### Text: @Yelp can we get the exact calculations for a business rating (for example if its 4 stars but actually 4.2) or do we use a 3rd party site?\n\n### Label: no complaint"},
        {"Tweet text":"@nationalgridus I have no water and the bill is current and paid. Can you do something about this?","ID":7,"Label":1,"text_label":"complaint","output":"### Text: @nationalgridus I have no water and the bill is current and paid. Can you do something about this?\n\n### Label: complaint"},
        {"Tweet text":"@JenniferTilly Merry Christmas to as well. You get more stunning every year ��","ID":9,"Label":2,"text_label":"no complaint","output":"### Text: @JenniferTilly Merry Christmas to as well. You get more stunning every year ��\n\n### Label: no complaint"}
    ]

$ oc apply -f training_data.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: trained-model
  namespace: kfto
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 50Gi

$ oc apply -f trainedmodelpvc.yaml

apiVersion: kubeflow.org/v1
kind: PyTorchJob
metadata:
  name: kfto-demo
  namespace: kfto
spec:
  pytorchReplicaSpecs:
    Master:
      replicas: 1
      restartPolicy: Never
      template:
        spec:
          containers:
            - env:
                - name: SFT_TRAINER_CONFIG_JSON_PATH
                  value: /etc/config/config.json
              image: 'quay.io/modh/fms-hf-tuning:release'
              imagePullPolicy: IfNotPresent
              name: pytorch
              volumeMounts:
                - mountPath: /etc/config
                  name: config-volume
                - mountPath: /data/input
                  name: dataset-volume
                - mountPath: /data/output
                  name: model-volume
          volumes:
            - configMap:
                items:
                  - key: config.json
                    path: config.json
                name: training-config
              name: config-volume
            - configMap:
                name: twitter-complaints
              name: dataset-volume
            - name: model-volume
              persistentVolumeClaim:
                claimName: trained-model
  runPolicy:
    suspend: false

$ oc apply -f pytorchjob.yaml

status:
 conditions:
   - lastTransitionTime: '2024-05-29T13:05:09Z'
     message: 'couldn''t assign flavors to pod set small-group-jobtest12: insufficient quota for nvidia.com/gpu in flavor default-flavor in ClusterQueue'