Add Infinity Instruct datagen

[rlakhtakia]

Sample Config:

load:
  type: constant
  stages:
  - rate: 1
    duration: 30
api: 
  type: completion
server:
  type: vllm
  model_name: meta-llama/Llama-3.1-8B-Instruct
  base_url: http://35.209.201.68:80
  ignore_eos: true
tokenizer:
  pretrained_model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
  token: <token>
storage:
  google_cloud_storage:
    bucket_name: inference-perf-results
data:
  type: infinity_instruct
  path: datasets/Infinity-Instruct_conversations.json

report:
  request_lifecycle:
    summary: true
    per_stage: true
    per_request: false
  prometheus:
    summary: true
    per_stage: false

Output Summary:

{
  "load_summary": {
    "count": 30,
    "schedule_delay": {
      "mean": 0.00114706016999359,
      "min": -0.000305728521198034,
      "p10": 0.000156184693332761,
      "median": 0.00133772497065365,
      "p90": 0.00198361872462556,
      "max": 0.00246874697040766
    }
  },
  "successes": {
    "count": 30,
    "latency": {
      "request_latency": {
        "mean": 1.53888443643227,
        "min": 0.317581715062261,
        "p10": 0.507761221088003,
        "median": 1.43748497095658,
        "p90": 2.31946949468693,
        "max": 5.95985660806764
      },
      "normalized_time_per_output_token": {
        "mean": 0.00747262092680917,
        "min": 0.00690199559705911,
        "p10": 0.00696805264314121,
        "median": 0.00725599521084779,
        "p90": 0.00808908041407286,
        "max": 0.00957854405777263
      },
      "time_per_output_token": null,
      "time_to_first_token": null,
      "inter_token_latency": null
    },
    "throughput": {
      "input_tokens_per_sec": 2217.61768148649,
      "output_tokens_per_sec": 209.355401079671,
      "total_tokens_per_sec": 2426.97308256616,
      "requests_per_sec": 0.980740479761106
    },
    "prompt_len": {
      "mean": 2261.16666666667,
      "min": 1086,
      "p10": 1301.2,
      "median": 1927,
      "p90": 3790.6,
      "max": 4088
    },
    "output_len": {
      "mean": 213.466666666667,
      "min": 34,
      "p10": 61.9,
      "median": 194,
      "p90": 325.4,
      "max": 856
    }
  },
  "failures": {
    "count": 0,
    "request_latency": null,
    "prompt_len": null
  }
}

[jjk-g]

/lgtm

[achandrasekar]

Can you fix this type check issue - https://github.com/kubernetes-sigs/inference-perf/actions/runs/17623590703/job/50172787339?pr=217?

[achandrasekar]

/remove-approve

[achandrasekar]

/lgtm
/approve

[k8s-ci-robot]

[APPROVALNOTIFIER] This PR is APPROVED

This pull-request has been approved by: achandrasekar, rlakhtakia

The full list of commands accepted by this bot can be found here.

The pull request process is described here

Details

Needs approval from an approver in each of these files:

• OWNERS [achandrasekar]

Approvers can indicate their approval by writing /approve in a comment
Approvers can cancel approval by writing /approve cancel in a comment