Evaluating AI systems

python3 -m venv .venv
source .venv/bin/activate

pip install \
    llama-stack \
    llama-stack-client \
    llama-stack-provider-lmeval

oc create route edge vllm --service=<VLLM_SERVICE> --port=<VLLM_PORT> -n <MODEL_NAMESPACE>

export TRUSTYAI_LM_EVAL_NAMESPACE=<MODEL_NAMESPACE>
export MODEL_ROUTE=$(oc get route -n "$TRUSTYAI_LM_EVAL_NAMESPACE" | awk '/predictor/{print $2; exit}')
export VLLM_URL="https://${MODEL_ROUTE}/v1/completions"

curl --create-dirs --output providers.d/remote/eval/trustyai_lmeval.yaml https://raw.githubusercontent.com/trustyai-explainability/llama-stack-provider-lmeval/refs/heads/main/providers.d/remote/eval/trustyai_lmeval.yaml

curl --create-dirs --output run.yaml https://raw.githubusercontent.com/trustyai-explainability/llama-stack-provider-lmeval/refs/heads/main/run.yaml

llama stack run run.yaml --image-type venv

import os
import subprocess

import logging

import time
import pprint

BASE_URL = "http://localhost:8321"

def create_http_client():
    from llama_stack_client import LlamaStackClient
    return LlamaStackClient(base_url=BASE_URL)

client = create_http_client()

benchmarks = client.benchmarks.list()

pprint.pprint(f"Available benchmarks: {benchmarks}")

client.benchmarks.register(
    benchmark_id="trustyai_lmeval::arc_easy",
    dataset_id="trustyai_lmeval::arc_easy",
    scoring_functions=["string"],
    provider_benchmark_id="string",
    provider_id="trustyai_lmeval",
     metadata={
        "tokenizer": "google/flan-t5-small",
        "tokenized_requests": False,
   }
)

benchmarks = client.benchmarks.list()
pprint.print(f"Available benchmarks: {benchmarks}")

job = client.eval.run_eval(
    benchmark_id="trustyai_lmeval::arc_easy",
    benchmark_config={
        "eval_candidate": {
            "type": "model",
            "model": "phi-3",
            "provider_id": "trustyai_lmeval",
            "sampling_params": {
                "temperature": 0.7,
                "top_p": 0.9,
                "max_tokens": 256
            },
        },
        "num_examples": 1000,
     },
)

print(f"Starting job '{job.job_id}'")

pprint.pprint(client.eval.jobs.retrieve(job_id=job.job_id, benchmark_id="trustyai_lmeval::arc_easy").scores)

oc apply -n <MODEL_NAMESPACE> -f - << EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
    name: my-pvc
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi
EOF

oc apply -n <MODEL_NAMESPACE> -f - << EOF
apiVersion: v1
kind: Pod
metadata:
  name: dataset-storage-pod
spec:
  containers:
  - name: dataset-container
    image: 'quay.io/prometheus/busybox:latest'
    command: ["/bin/sh", "-c", "sleep 3600"]
    volumeMounts:
    - mountPath: "/data/upload_files"
      name: dataset-storage
  volumes:
  - name: dataset-storage
    persistentVolumeClaim:
      claimName: my-pvc
EOF

oc cp example-dk-bench-input-bmo.jsonl dataset-storage-pod:/data/upload_files/example-dk-bench-input-bmo.jsonl -n <MODEL_NAMESPACE>

job = client.eval.run_eval(
    benchmark_id="trustyai_lmeval::dk-bench",
    benchmark_config={
        "eval_candidate": {
            "type": "model",
            "model": "phi-3",
            "provider_id": "trustyai_lmeval",
            "sampling_params": {
                "temperature": 0.7,
                "top_p": 0.9,
                "max_tokens": 256
            },
        },
        "num_examples": 1000,
     },
)

print(f"Starting job '{job.job_id}'")

import time
def get_job_status(job_id, benchmark_id):
    return client.eval.jobs.status(job_id=job_id, benchmark_id=benchmark_id)

while True:
    job = get_job_status(job_id=job.job_id, benchmark_id="trustyai_lmeval::dk-bench")
    print(job)

    if job.status in ['failed', 'completed']:
        print(f"Job ended with status: {job.status}")
        break

    time.sleep(20)

spec:
  trustyai:
    managementState: Managed
  llamastack:
    managementState: Managed
  kserve:
    defaultDeploymentMode: RawDeployment
    managementState: Managed
    nim:
      managementState: Managed
    rawDeploymentServiceConfig: Headless
  serving:
    ingressGateway:
      certificate:
        type: OpenshiftDefaultIngress
    managementState: Removed
    name: knative-serving
  serviceMesh:
    managementState: Removed

PROJECT_NAME="lls-minimal-example"
oc new-project $PROJECT_NAME

cat <<EOF | oc apply -f -
kind: ConfigMap
apiVersion: v1
metadata:
  name: fms-orchestr8-config-nlp
data:
  config.yaml: |
    detectors:
      regex:
        type: text_contents
        service:
          hostname: "127.0.0.1"
          port: 8080
        chunker_id: whole_doc_chunker
        default_threshold: 0.5
---
apiVersion: trustyai.opendatahub.io/v1alpha1
kind: GuardrailsOrchestrator
metadata:
  name: guardrails-orchestrator
spec:
  orchestratorConfig: "fms-orchestr8-config-nlp"
  enableBuiltInDetectors: true
  enableGuardrailsGateway: false
  replicas: 1
EOF

apiVersion: llamastack.io/v1alpha1
kind: LlamaStackDistribution
metadata:
  name: llamastackdistribution-sample
  namespace: <PROJECT_NAMESPACE>
spec:
  replicas: 1
  server:
    containerSpec:
      env:
        - name: VLLM_URL
          value: '${VLLM_URL}'
        - name: INFERENCE_MODEL
          value: '${INFERENCE_MODEL}'
        - name: MILVUS_DB_PATH
          value: '~/.llama/milvus.db'
        - name: VLLM_TLS_VERIFY
          value: 'false'
        - name: FMS_ORCHESTRATOR_URL
          value: '${FMS_ORCHESTRATOR_URL}'
      name: llama-stack
      port: 8321
    distribution:
      name: rh-dev
    storage:
      size: 20Gi

oc -n $PROJECT_NAME port-forward svc/llama-stack 8321:8321

curl -X POST http://localhost:8321/v1/shields \
  -H 'Content-Type: application/json' \
  -d '{
    "shield_id": "regex_detector",
    "provider_shield_id": "regex_detector",
    "provider_id": "trustyai_fms",
    "params": {
      "type": "content",
      "confidence_threshold": 0.5,
      "message_types": ["system", "user"],
      "detectors": {
        "regex": {
          "detector_params": {
            "regex": ["email", "us-social-security-number", "credit-card"]
          }
        }
      }
    }
  }'

curl -s http://localhost:8321/v1/shields | jq '.'

{
  "data": [
    {
      "identifier": "regex_detector",
      "provider_resource_id": "regex_detector",
      "provider_id": "trustyai_fms",
      "type": "shield",
      "params": {
        "type": "content",
        "confidence_threshold": 0.5,
        "message_types": [
          "system",
          "user"
        ],
        "detectors": {
          "regex": {
            "detector_params": {
              "regex": [
                "email",
                "us-social-security-number",
                "credit-card"
              ]
            }
          }
        }
      }
    }
  ]
}

$ oc login <openshift_cluster_url> -u <username> -p <password>

$ oc project <project_name>

apiVersion: v1
kind: ConfigMap
metadata:
  name: ragas-inline-config
  namespace: <project_name>
data:
  EMBEDDING_MODEL: "all-MiniLM-L6-v2"

$ oc apply -f ragas-inline-config.yaml

apiVersion: llamastack.trustyai.opendatahub.io/v1alpha1
kind: LlamaStackDistribution
metadata:
  name: llama-stack-ragas-inline
  namespace: <project_name>
spec:
  replicas: 1
  server:
    containerSpec:
      env:
# ...
      - name: VLLM_URL
        value: <model_url>
      - name: VLLM_API_TOKEN
        value: <model_api_token (if necessary)>
      - name: INFERENCE_MODEL
        value: <model_name>
      - name: MILVUS_DB_PATH
        value: ~/.llama/milvus.db
      - name: VLLM_TLS_VERIFY
        value: "false"
      - name: FMS_ORCHESTRATOR_URL
        value: http://localhost:123
      - name: EMBEDDING_MODEL
        value: granite-embedding-125m
# ...

$ oc apply -f llama-stack-ragas-inline.yaml

$ oc get pods -w

$ oc login <openshift_cluster_url> -u <username> -p <password>

$ oc project <project_name>

$ oc create secret generic "<ragas_s3_credentials>" \
  --from-literal=AWS_ACCESS_KEY_ID=<your_access_key> \
  --from-literal=AWS_SECRET_ACCESS_KEY=<your_secret_key> \
  --from-literal=AWS_DEFAULT_REGION=<your_region>

$ curl -k -H "Authorization: Bearer $KUBEFLOW_PIPELINES_TOKEN" \
 https://$KUBEFLOW_PIPELINES_ENDPOINT/apis/v1beta1/healthz

$ export INFERENCE_MODEL="llama-3-2-3b"
$ export VLLM_URL="https://llama-32-3b-instruct-predictor:8443/v1"
$ export VLLM_TLS_VERIFY="false"  # Use "true" in production
$ export VLLM_API_TOKEN="<token_identifier>"

$ oc create secret generic llama-stack-inference-model-secret \
  --from-literal INFERENCE_MODEL="$INFERENCE_MODEL" \
  --from-literal VLLM_URL="$VLLM_URL" \
  --from-literal VLLM_TLS_VERIFY="$VLLM_TLS_VERIFY" \
  --from-literal VLLM_API_TOKEN="$VLLM_API_TOKEN"

$ oc get routes -A | grep -i pipeline

apiVersion: v1
kind: ConfigMap
metadata:
  name: kubeflow-ragas-config
  namespace: <project_name>
data:
  EMBEDDING_MODEL: "all-MiniLM-L6-v2"
  KUBEFLOW_LLAMA_STACK_URL: "http://$<distribution_name>-service.$<your_namespace>.svc.cluster.local:$<port>"
  KUBEFLOW_PIPELINES_ENDPOINT: "https://<kfp_endpoint>"
  KUBEFLOW_NAMESPACE: "<project_name>"
  KUBEFLOW_BASE_IMAGE: "quay.io/rhoai/odh-trustyai-ragas-lls-provider-dsp-rhel9:rhoai-3.0"
  KUBEFLOW_RESULTS_S3_PREFIX: "s3://<bucket_name>/ragas-results"
  KUBEFLOW_S3_CREDENTIALS_SECRET_NAME: "<ragas_s3_credentials>"

$ oc apply -f kubeflow-ragas-config.yaml

apiVersion: llamastack.io/v1alpha1
kind: LlamaStackDistribution
metadata:
  name: llama-stack-pod
spec:
  replicas: 1
  server:
    containerSpec:
      resources:
        requests:
          cpu: 4
          memory: "12Gi"
        limits:
          cpu: 6
          memory: "14Gi"
      env:
        - name: INFERENCE_MODEL
          valueFrom:
            secretKeyRef:
              key: INFERENCE_MODEL
              name: llama-stack-inference-model-secret
              optional: true
        - name: VLLM_MAX_TOKENS
          value: "4096"
        - name: VLLM_URL
          valueFrom:
            secretKeyRef:
              key: VLLM_URL
              name: llama-stack-inference-model-secret
              optional: true
        - name: VLLM_TLS_VERIFY
          valueFrom:
            secretKeyRef:
              key: VLLM_TLS_VERIFY
              name: llama-stack-inference-model-secret
              optional: true
        - name: VLLM_API_TOKEN
          valueFrom:
            secretKeyRef:
              key: VLLM_API_TOKEN
              name: llama-stack-inference-model-secret
              optional: true
        - name: MILVUS_DB_PATH
          value: ~/milvus.db
        - name: FMS_ORCHESTRATOR_URL
          value: "http://localhost"
        - name: KUBEFLOW_PIPELINES_ENDPOINT
          valueFrom:
            configMapKeyRef:
              key: KUBEFLOW_PIPELINES_ENDPOINT
              name: kubeflow-ragas-config
              optional: true
        - name: KUBEFLOW_NAMESPACE
          valueFrom:
            configMapKeyRef:
              key: KUBEFLOW_NAMESPACE
              name: kubeflow-ragas-config
              optional: true
        - name: KUBEFLOW_BASE_IMAGE
          valueFrom:
            configMapKeyRef:
              key: KUBEFLOW_BASE_IMAGE
              name: kubeflow-ragas-config
              optional: true
        - name: KUBEFLOW_LLAMA_STACK_URL
          valueFrom:
            configMapKeyRef:
              key: KUBEFLOW_LLAMA_STACK_URL
              name: kubeflow-ragas-config
              optional: true
        - name: KUBEFLOW_RESULTS_S3_PREFIX
          valueFrom:
            configMapKeyRef:
              key: KUBEFLOW_RESULTS_S3_PREFIX
              name: kubeflow-ragas-config
              optional: true
        - name: KUBEFLOW_S3_CREDENTIALS_SECRET_NAME
          valueFrom:
            configMapKeyRef:
              key: KUBEFLOW_S3_CREDENTIALS_SECRET_NAME
              name: kubeflow-ragas-config
              optional: true
        - name: EMBEDDING_MODEL
          valueFrom:
            configMapKeyRef:
              key: EMBEDDING_MODEL
              name: kubeflow-ragas-config
              optional: true
        - name: KUBEFLOW_PIPELINES_TOKEN
          valueFrom:
            secretKeyRef:
              key: KUBEFLOW_PIPELINES_TOKEN
              name: kubeflow-pipelines-token
              optional: true
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef:
              key: AWS_ACCESS_KEY_ID
              name: "<ragas_s3_credentials>"
              optional: true
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              key: AWS_SECRET_ACCESS_KEY
              name: "<ragas_s3_credentials>"
              optional: true
        - name: AWS_DEFAULT_REGION
          valueFrom:
            secretKeyRef:
              key: AWS_DEFAULT_REGION
              name: "<ragas_s3_credentials>"
              optional: true
      name: llama-stack
      port: 8321
    distribution:
      name: rh-dev

$ oc apply -f llama-stack-ragas-remote.yaml

$ oc get pods -w