worker-config.json

{
  "versions": {
    "0.10.0": {
      "imageName": "runpod/worker-v1-vllm:v2.8.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.9.0": {
      "imageName": "runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.9.1": {
      "imageName": "runpod/worker-v1-vllm:v2.7.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.9.0": {
      "imageName": "runpod/worker-v1-vllm:v2.6.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.8.5": {
      "imageName": "runpod/worker-v1-vllm:v2.5.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.8.4": {
      "imageName": "runpod/worker-v1-vllm:v2.4.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.8.3": {
      "imageName": "runpod/worker-v1-vllm:v2.3.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.8.2": {
      "imageName": "runpod/worker-v1-vllm:v2.2.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.7.3": {
      "imageName": "runpod/worker-v1-vllm:v2.1.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.6.6": {
      "imageName": "runpod/worker-v1-vllm:v1.8.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.7.0": {
      "imageName": "runpod/worker-v1-vllm:v1.9.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.6.4": {
      "imageName": "runpod/worker-v1-vllm:v1.7.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    },
    "0.6.3": {
      "imageName": "runpod/worker-v1-vllm:v1.6.0stable-cuda12.1.0",
      "minimumCudaVersion": "12.1",
      "categories": [
        {
          "title": "LLM Settings",
          "settings": [
            "TOKENIZER", "TOKENIZER_MODE", "SKIP_TOKENIZER_INIT", "TRUST_REMOTE_CODE",
            "DOWNLOAD_DIR", "LOAD_FORMAT", "DTYPE", "KV_CACHE_DTYPE", "QUANTIZATION_PARAM_PATH",
            "MAX_MODEL_LEN", "GUIDED_DECODING_BACKEND", "DISTRIBUTED_EXECUTOR_BACKEND",
            "WORKER_USE_RAY", "RAY_WORKERS_USE_NSIGHT", "PIPELINE_PARALLEL_SIZE",
            "TENSOR_PARALLEL_SIZE", "MAX_PARALLEL_LOADING_WORKERS", "ENABLE_PREFIX_CACHING",
            "DISABLE_SLIDING_WINDOW", "NUM_LOOKAHEAD_SLOTS",
            "SEED", "NUM_GPU_BLOCKS_OVERRIDE", "MAX_NUM_BATCHED_TOKENS", "MAX_NUM_SEQS",
            "MAX_LOGPROBS", "DISABLE_LOG_STATS", "QUANTIZATION", "ROPE_SCALING", "ROPE_THETA",
            "TOKENIZER_POOL_SIZE", "TOKENIZER_POOL_TYPE", "TOKENIZER_POOL_EXTRA_CONFIG",
            "ENABLE_LORA", "MAX_LORAS", "MAX_LORA_RANK", "LORA_EXTRA_VOCAB_SIZE",
            "LORA_DTYPE", "LONG_LORA_SCALING_FACTORS", "MAX_CPU_LORAS", "FULLY_SHARDED_LORAS",
            "DEVICE", "SCHEDULER_DELAY_FACTOR", "ENABLE_CHUNKED_PREFILL", "SPECULATIVE_MODEL",
            "NUM_SPECULATIVE_TOKENS", "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
            "SPECULATIVE_MAX_MODEL_LEN", "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
            "NGRAM_PROMPT_LOOKUP_MAX", "NGRAM_PROMPT_LOOKUP_MIN", "SPEC_DECODING_ACCEPTANCE_METHOD",
            "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD", "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
            "MODEL_LOADER_EXTRA_CONFIG", "PREEMPTION_MODE", "PREEMPTION_CHECK_PERIOD",
            "PREEMPTION_CPU_CAPACITY", "MAX_LOG_LEN", "DISABLE_LOGGING_REQUEST",
            "ENABLE_AUTO_TOOL_CHOICE", "TOOL_CALL_PARSER"
          ]
        },
        {
          "title": "Tokenizer Settings",
          "settings": [
            "TOKENIZER_NAME", "TOKENIZER_REVISION", "CUSTOM_CHAT_TEMPLATE"
          ]
        },
        {
          "title": "System Settings",
          "settings": [
            "GPU_MEMORY_UTILIZATION", "MAX_PARALLEL_LOADING_WORKERS", "BLOCK_SIZE",
            "SWAP_SPACE", "ENFORCE_EAGER", "MAX_SEQ_LEN_TO_CAPTURE", "DISABLE_CUSTOM_ALL_REDUCE"
          ]
        },
        {
          "title": "Streaming Settings",
          "settings": [
            "DEFAULT_BATCH_SIZE", "DEFAULT_MIN_BATCH_SIZE", "DEFAULT_BATCH_SIZE_GROWTH_FACTOR"
          ]
        },
        {
          "title": "OpenAI Settings",
          "settings": [
            "RAW_OPENAI_OUTPUT", "OPENAI_RESPONSE_ROLE", "OPENAI_SERVED_MODEL_NAME_OVERRIDE"
          ]
        },
        {
          "title": "Serverless Settings",
          "settings": [
            "MAX_CONCURRENCY", "DISABLE_LOG_STATS", "DISABLE_LOG_REQUESTS"
          ]
        }
      ]
    }
  },
  "schema": {
    "TOKENIZER": {
      "env_var_name": "TOKENIZER",
      "value": "",
      "title": "Tokenizer",
      "description": "Name or path of the Hugging Face tokenizer to use.",
      "required": false,
      "type": "text"
    },
    "TOKENIZER_MODE": {
      "env_var_name": "TOKENIZER_MODE",
      "value": "auto",
      "title": "Tokenizer Mode",
      "description": "The tokenizer mode.",
      "required": false,
      "type": "select",
      "options": [
        { "value": "auto", "label": "auto" },
        { "value": "slow", "label": "slow" }
      ]
    },
    "SKIP_TOKENIZER_INIT": {
      "env_var_name": "SKIP_TOKENIZER_INIT",
      "value": false,
      "title": "Skip Tokenizer Init",
      "description": "Skip initialization of tokenizer and detokenizer.",
      "required": false,
      "type": "toggle"
    },
    "TRUST_REMOTE_CODE": {
      "env_var_name": "TRUST_REMOTE_CODE",
      "value": false,
      "title": "Trust Remote Code",
      "description": "Trust remote code from Hugging Face.",
      "required": false,
      "type": "toggle"
    },
    "DOWNLOAD_DIR": {
      "env_var_name": "DOWNLOAD_DIR",
      "value": "",
      "title": "Download Directory",
      "description": "Directory to download and load the weights.",
      "required": false,
      "type": "text"
    },
    "LOAD_FORMAT": {
      "env_var_name": "LOAD_FORMAT",
      "value": "auto",
      "title": "Load Format",
      "description": "The format of the model weights to load.",
      "required": false,
      "type": "select",
      "options": [
        { "value": "auto", "label": "auto" },
        { "value": "pt", "label": "pt" },
        { "value": "safetensors", "label": "safetensors" },
        { "value": "npcache", "label": "npcache" },
        { "value": "dummy", "label": "dummy" },
        { "value": "tensorizer", "label": "tensorizer" },
        { "value": "bitsandbytes", "label": "bitsandbytes" }
      ]
    },
    "DTYPE": {
      "env_var_name": "DTYPE",
      "value": "auto",
      "title": "Data Type",
      "description": "Data type for model weights and activations.",
      "required": false,
      "type": "select",
      "options": [
        { "value": "auto", "label": "auto" },
        { "value": "half", "label": "half" },
        { "value": "float16", "label": "float16" },
        { "value": "bfloat16", "label": "bfloat16" },
        { "value": "float", "label": "float" },
        { "value": "float32", "label": "float32" }
      ]
    },
    "KV_CACHE_DTYPE": {
      "env_var_name": "KV_CACHE_DTYPE",
      "value": "auto",
      "title": "KV Cache Data Type",
      "description": "Data type for KV cache storage.",
      "required": false,
      "type": "select",
      "options": [
        { "value": "auto", "label": "auto" },
        { "value": "fp8", "label": "fp8" }
      ]
    },
    "QUANTIZATION_PARAM_PATH": {
      "env_var_name": "QUANTIZATION_PARAM_PATH",
      "value": "",
      "title": "Quantization Param Path",
      "description": "Path to the JSON file containing the KV cache scaling factors.",
      "required": false,
      "type": "text"
    },
    "MAX_MODEL_LEN": {
      "env_var_name": "MAX_MODEL_LEN",
      "value": "",
      "title": "Max Model Length",
      "description": "Model context length.",
      "required": false,
      "type": "number"
    },
    "GUIDED_DECODING_BACKEND": {
      "env_var_name": "GUIDED_DECODING_BACKEND",
      "value": "outlines",
      "title": "Guided Decoding Backend",
      "description": "Which engine will be used for guided decoding by default.",
      "required": false,
      "type": "select",
      "options": [
        { "value": "outlines", "label": "outlines" },
        { "value": "lm-format-enforcer", "label": "lm-format-enforcer" }
      ]
    },
    "DISTRIBUTED_EXECUTOR_BACKEND": {
    "env_var_name": "DISTRIBUTED_EXECUTOR_BACKEND",
    "value": "",
    "title": "Distributed Executor Backend",
    "description": "Backend to use for distributed serving.",
    "required": false,
    "type": "select",
    "options": [
      { "value": "ray", "label": "ray" },
      { "value": "mp", "label": "mp" }
    ]
  },
  "WORKER_USE_RAY": {
    "env_var_name": "WORKER_USE_RAY",
    "value": false,
    "title": "Worker Use Ray",
    "description": "Deprecated, use --distributed-executor-backend=ray.",
    "required": false,
    "type": "toggle"
  },
  "RAY_WORKERS_USE_NSIGHT": {
    "env_var_name": "RAY_WORKERS_USE_NSIGHT",
    "value": false,
    "title": "Ray Workers Use Nsight",
    "description": "If specified, use nsight to profile Ray workers.",
    "required": false,
    "type": "toggle"
  },
  "PIPELINE_PARALLEL_SIZE": {
    "env_var_name": "PIPELINE_PARALLEL_SIZE",
    "value": 1,
    "title": "Pipeline Parallel Size",
    "description": "Number of pipeline stages.",
    "required": false,
    "type": "number"
  },
  "TENSOR_PARALLEL_SIZE": {
    "env_var_name": "TENSOR_PARALLEL_SIZE",
    "value": 1,
    "title": "Tensor Parallel Size",
    "description": "Number of tensor parallel replicas.",
    "required": false,
    "type": "number"
  },
  "MAX_PARALLEL_LOADING_WORKERS": {
    "env_var_name": "MAX_PARALLEL_LOADING_WORKERS",
    "value": "",
    "title": "Max Parallel Loading Workers",
    "description": "Load model sequentially in multiple batches.",
    "required": false,
    "type": "number"
  },
  "ENABLE_PREFIX_CACHING": {
    "env_var_name": "ENABLE_PREFIX_CACHING",
    "value": false,
    "title": "Enable Prefix Caching",
    "description": "Enables automatic prefix caching.",
    "required": false,
    "type": "toggle"
  },
  "DISABLE_SLIDING_WINDOW": {
    "env_var_name": "DISABLE_SLIDING_WINDOW",
    "value": false,
    "title": "Disable Sliding Window",
    "description": "Disables sliding window, capping to sliding window size.",
    "required": false,
    "type": "toggle"
  },
  "USE_V2_BLOCK_MANAGER": {
    "env_var_name": "USE_V2_BLOCK_MANAGER",
    "value": false,
    "title": "Use V2 Block Manager",
    "description": "Use BlockSpaceMangerV2.",
    "required": false,
    "type": "toggle"
  },
  "NUM_LOOKAHEAD_SLOTS": {
    "env_var_name": "NUM_LOOKAHEAD_SLOTS",
    "value": 0,
    "title": "Num Lookahead Slots",
    "description": "Experimental scheduling config necessary for speculative decoding.",
    "required": false,
    "type": "number"
  },
  "SEED": {
    "env_var_name": "SEED",
    "value": 0,
    "title": "Seed",
    "description": "Random seed for operations.",
    "required": false,
    "type": "number"
  },
  "NUM_GPU_BLOCKS_OVERRIDE": {
    "env_var_name": "NUM_GPU_BLOCKS_OVERRIDE",
    "value": "",
    "title": "Num GPU Blocks Override",
    "description": "If specified, ignore GPU profiling result and use this number of GPU blocks.",
    "required": false,
    "type": "number"
  },
  "MAX_NUM_BATCHED_TOKENS": {
    "env_var_name": "MAX_NUM_BATCHED_TOKENS",
    "value": "",
    "title": "Max Num Batched Tokens",
    "description": "Maximum number of batched tokens per iteration.",
    "required": false,
    "type": "number"
  },
  "MAX_NUM_SEQS": {
    "env_var_name": "MAX_NUM_SEQS",
    "value": 256,
    "title": "Max Num Seqs",
    "description": "Maximum number of sequences per iteration.",
    "required": false,
    "type": "number"
  },
  "MAX_LOGPROBS": {
    "env_var_name": "MAX_LOGPROBS",
    "value": 20,
    "title": "Max Logprobs",
    "description": "Max number of log probs to return when logprobs is specified in SamplingParams.",
    "required": false,
    "type": "number"
  },
  "DISABLE_LOG_STATS": {
    "env_var_name": "DISABLE_LOG_STATS",
    "value": false,
    "title": "Disable Log Stats",
    "description": "Disable logging statistics.",
    "required": false,
    "type": "toggle"
  },
  "QUANTIZATION": {
    "env_var_name": "QUANTIZATION",
    "value": "",
    "title": "Quantization",
    "description": "Method used to quantize the weights.\nif the `Load Format` is 'bitsandbytes' then `Quantization` will be forced to 'bitsandbytes'",
    "required": false,
    "type": "select",
    "options": [
      { "value": "None", "label": "None" },
      { "value": "awq", "label": "AWQ" },
      { "value": "squeezellm", "label": "SqueezeLLM" },
      { "value": "gptq", "label": "GPTQ" },
      { "value": "bitsandbytes", "label": "bitsandbytes" }
    ]
  },
  "ROPE_SCALING": {
    "env_var_name": "ROPE_SCALING",
    "value": "",
    "title": "RoPE Scaling",
    "description": "RoPE scaling configuration in JSON format.",
    "required": false,
    "type": "text"
  },
  "ROPE_THETA": {
    "env_var_name": "ROPE_THETA",
    "value": "",
    "title": "RoPE Theta",
    "description": "RoPE theta. Use with rope_scaling.",
    "required": false,
    "type": "number"
  },
  "TOKENIZER_POOL_SIZE": {
    "env_var_name": "TOKENIZER_POOL_SIZE",
    "value": 0,
    "title": "Tokenizer Pool Size",
    "description": "Size of tokenizer pool to use for asynchronous tokenization.",
    "required": false,
    "type": "number"
  },
  "TOKENIZER_POOL_TYPE": {
    "env_var_name": "TOKENIZER_POOL_TYPE",
    "value": "ray",
    "title": "Tokenizer Pool Type",
    "description": "Type of tokenizer pool to use for asynchronous tokenization.",
    "required": false,
    "type": "text"
  },
  "TOKENIZER_POOL_EXTRA_CONFIG": {
    "env_var_name": "TOKENIZER_POOL_EXTRA_CONFIG",
    "value": "",
    "title": "Tokenizer Pool Extra Config",
    "description": "Extra config for tokenizer pool.",
    "required": false,
    "type": "text"
  },
  "ENABLE_LORA": {
    "env_var_name": "ENABLE_LORA",
    "value": false,
    "title": "Enable LoRA",
    "description": "If True, enable handling of LoRA adapters.",
    "required": false,
    "type": "toggle"
  },
  "MAX_LORAS": {
    "env_var_name": "MAX_LORAS",
    "value": 1,
    "title": "Max LoRAs",
    "description": "Max number of LoRAs in a single batch.",
    "required": false,
    "type": "number"
  },
  "MAX_LORA_RANK": {
    "env_var_name": "MAX_LORA_RANK",
    "value": 16,
    "title": "Max LoRA Rank",
    "description": "Max LoRA rank.",
    "required": false,
    "type": "number"
  },
  "LORA_EXTRA_VOCAB_SIZE": {
    "env_var_name": "LORA_EXTRA_VOCAB_SIZE",
    "value": 256,
    "title": "LoRA Extra Vocab Size",
    "description": "Maximum size of extra vocabulary for LoRA adapters.",
    "required": false,
    "type": "number"
  },
  "LORA_DTYPE": {
    "env_var_name": "LORA_DTYPE",
    "value": "auto",
    "title": "LoRA Data Type",
    "description": "Data type for LoRA.",
    "required": false,
    "type": "select",
    "options": [
      { "value": "auto", "label": "auto" },
      { "value": "float16", "label": "float16" },
      { "value": "bfloat16", "label": "bfloat16" },
      { "value": "float32", "label": "float32" }
    ]
  },
  "LONG_LORA_SCALING_FACTORS": {
    "env_var_name": "LONG_LORA_SCALING_FACTORS",
    "value": "",
    "title": "Long LoRA Scaling Factors",
    "description": "Specify multiple scaling factors for LoRA adapters.",
    "required": false,
    "type": "text"
  },
  "MAX_CPU_LORAS": {
    "env_var_name": "MAX_CPU_LORAS",
    "value": "",
    "title": "Max CPU LoRAs",
    "description": "Maximum number of LoRAs to store in CPU memory.",
    "required": false,
    "type": "number"
  },
  "FULLY_SHARDED_LORAS": {
    "env_var_name": "FULLY_SHARDED_LORAS",
    "value": false,
    "title": "Fully Sharded LoRAs",
    "description": "Enable fully sharded LoRA layers.",
    "required": false,
    "type": "toggle"
  },
  "DEVICE": {
    "env_var_name": "DEVICE",
    "value": "auto",
    "title": "Device",
    "description": "Device type for vLLM execution.",
    "required": false,
    "type": "select",
    "options": [
      { "value": "auto", "label": "auto" },
      { "value": "cuda", "label": "cuda" },
      { "value": "neuron", "label": "neuron" },
      { "value": "cpu", "label": "cpu" },
      { "value": "openvino", "label": "openvino" },
      { "value": "tpu", "label": "tpu" },
      { "value": "xpu", "label": "xpu" }
    ]
  },
  "SCHEDULER_DELAY_FACTOR": {
    "env_var_name": "SCHEDULER_DELAY_FACTOR",
    "value": 0.0,
    "title": "Scheduler Delay Factor",
    "description": "Apply a delay before scheduling next prompt.",
    "required": false,
    "type": "number"
  },
  "ENABLE_CHUNKED_PREFILL": {
    "env_var_name": "ENABLE_CHUNKED_PREFILL",
    "value": false,
    "title": "Enable Chunked Prefill",
    "description": "Enable chunked prefill requests.",
    "required": false,
    "type": "toggle"
  },
  "SPECULATIVE_MODEL": {
    "env_var_name": "SPECULATIVE_MODEL",
    "value": "",
    "title": "Speculative Model",
    "description": "The name of the draft model to be used in speculative decoding.",
    "required": false,
    "type": "text"
  },
  "NUM_SPECULATIVE_TOKENS": {
    "env_var_name": "NUM_SPECULATIVE_TOKENS",
    "value": "",
    "title": "Num Speculative Tokens",
    "description": "The number of speculative tokens to sample from the draft model.",
    "required": false,
    "type": "number"
  },
  "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE": {
    "env_var_name": "SPECULATIVE_DRAFT_TENSOR_PARALLEL_SIZE",
    "value": "",
    "title": "Speculative Draft Tensor Parallel Size",
    "description": "Number of tensor parallel replicas for the draft model.",
    "required": false,
    "type": "number"
  },
  "SPECULATIVE_MAX_MODEL_LEN": {
    "env_var_name": "SPECULATIVE_MAX_MODEL_LEN",
    "value": "",
    "title": "Speculative Max Model Length",
    "description": "The maximum sequence length supported by the draft model.",
    "required": false,
    "type": "number"
  },
  "SPECULATIVE_DISABLE_BY_BATCH_SIZE": {
    "env_var_name": "SPECULATIVE_DISABLE_BY_BATCH_SIZE",
    "value": "",
    "title": "Speculative Disable by Batch Size",
    "description": "Disable speculative decoding if the number of enqueue requests is larger than this value.",
    "required": false,
    "type": "number"
  },
  "NGRAM_PROMPT_LOOKUP_MAX": {
    "env_var_name": "NGRAM_PROMPT_LOOKUP_MAX",
    "value": "",
    "title": "Ngram Prompt Lookup Max",
    "description": "Max size of window for ngram prompt lookup in speculative decoding.",
    "required": false,
    "type": "number"
  },
  "NGRAM_PROMPT_LOOKUP_MIN": {
    "env_var_name": "NGRAM_PROMPT_LOOKUP_MIN",
    "value": "",
    "title": "Ngram Prompt Lookup Min",
    "description": "Min size of window for ngram prompt lookup in speculative decoding.",
    "required": false,
    "type": "number"
  },
  "SPEC_DECODING_ACCEPTANCE_METHOD": {
    "env_var_name": "SPEC_DECODING_ACCEPTANCE_METHOD",
    "value": "rejection_sampler",
    "title": "Speculative Decoding Acceptance Method",
    "description": "Specify the acceptance method for draft token verification in speculative decoding.",
    "required": false,
    "type": "select",
    "options": [
      { "value": "rejection_sampler", "label": "rejection_sampler" },
      { "value": "typical_acceptance_sampler", "label": "typical_acceptance_sampler" }
    ]
  },
  "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD": {
    "env_var_name": "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_THRESHOLD",
    "value": "",
    "title": "Typical Acceptance Sampler Posterior Threshold",
    "description": "Set the lower bound threshold for the posterior probability of a token to be accepted.",
    "required": false,
    "type": "number"
  },
  "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA": {
    "env_var_name": "TYPICAL_ACCEPTANCE_SAMPLER_POSTERIOR_ALPHA",
    "value": "",
    "title": "Typical Acceptance Sampler Posterior Alpha",
    "description": "A scaling factor for the entropy-based threshold for token acceptance.",
    "required": false,
    "type": "number"
  },
  "MODEL_LOADER_EXTRA_CONFIG": {
    "env_var_name": "MODEL_LOADER_EXTRA_CONFIG",
    "value": "",
    "title": "Model Loader Extra Config",
    "description": "Extra config for model loader.",
    "required": false,
    "type": "text"
  },
  "PREEMPTION_MODE": {
    "env_var_name": "PREEMPTION_MODE",
    "value": "",
    "title": "Preemption Mode",
    "description": "If 'recompute', the engine performs preemption-aware recomputation. If 'save', the engine saves activations into the CPU memory as preemption happens.",
    "required": false,
    "type": "text"
  },
  "PREEMPTION_CHECK_PERIOD": {
    "env_var_name": "PREEMPTION_CHECK_PERIOD",
    "value": 1.0,
    "title": "Preemption Check Period",
    "description": "How frequently the engine checks if a preemption happens.",
    "required": false,
    "type": "number"
  },
  "PREEMPTION_CPU_CAPACITY": {
    "env_var_name": "PREEMPTION_CPU_CAPACITY",
    "value": 2,
    "title": "Preemption CPU Capacity",
    "description": "The percentage of CPU memory used for the saved activations.",
    "required": false,
    "type": "number"
  },
  "MAX_LOG_LEN": {
    "env_var_name": "MAX_LOG_LEN",
    "value": "",
    "title": "Max Log Length",
    "description": "Max number of characters or ID numbers being printed in log.",
    "required": false,
    "type": "number"
  },
  "DISABLE_LOGGING_REQUEST": {
    "env_var_name": "DISABLE_LOGGING_REQUEST",
    "value": false,
    "title": "Disable Logging Request",
    "description": "Disable logging requests.",
    "required": false,
    "type": "toggle"
  },
  "TOKENIZER_NAME": {
    "env_var_name": "TOKENIZER_NAME",
    "value": "",
    "title": "Tokenizer Name",
    "description": "Tokenizer repo to use a different tokenizer than the model's default",
    "required": false,
    "type": "text"
  },
  "TOKENIZER_REVISION": {
    "env_var_name": "TOKENIZER_REVISION",
    "value": "",
    "title": "Tokenizer Revision",
    "description": "Tokenizer revision to load",
    "required": false,
    "type": "text"
  },
  "CUSTOM_CHAT_TEMPLATE": {
    "env_var_name": "CUSTOM_CHAT_TEMPLATE",
    "value": "",
    "title": "Custom Chat Template",
    "description": "Custom chat jinja template",
    "required": false,
    "type": "text"
  },
  "GPU_MEMORY_UTILIZATION": {
    "env_var_name": "GPU_MEMORY_UTILIZATION",
    "value": "0.95",
    "title": "GPU Memory Utilization",
    "description": "Sets GPU VRAM utilization",
    "required": false,
    "type": "number"
  },
  "BLOCK_SIZE": {
    "env_var_name": "BLOCK_SIZE",
    "value": "16",
    "title": "Block Size",
    "description": "Token block size for contiguous chunks of tokens",
    "required": false,
    "type": "number"
  },
  "SWAP_SPACE": {
    "env_var_name": "SWAP_SPACE",
    "value": "4",
    "title": "Swap Space",
    "description": "CPU swap space size (GiB) per GPU",
    "required": false,
    "type": "number"
  },
  "ENFORCE_EAGER": {
    "env_var_name": "ENFORCE_EAGER",
    "value": false,
    "title": "Enforce Eager",
    "description": "Always use eager-mode PyTorch. If False (0), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility",
    "required": false,
    "type": "toggle"
  },
  "MAX_SEQ_LEN_TO_CAPTURE": {
    "env_var_name": "MAX_SEQ_LEN_TO_CAPTURE",
    "value": "8192",
    "title": "CUDA Graph Max Content Length",
    "description": "Maximum context length covered by CUDA graphs. If a sequence has context length larger than this, we fall back to eager mode",
    "required": false,
    "type": "number"
  },
  "DISABLE_CUSTOM_ALL_REDUCE": {
    "env_var_name": "DISABLE_CUSTOM_ALL_REDUCE",
    "value": false,
    "title": "Disable Custom All Reduce",
    "description": "Enables or disables custom all reduce",
    "required": false,
    "type": "toggle"
  },
  "DEFAULT_BATCH_SIZE": {
    "env_var_name": "DEFAULT_BATCH_SIZE",
    "value": "50",
    "title": "Default Final Batch Size",
    "description": "Default and Maximum batch size for token streaming to reduce HTTP calls",
    "required": false,
    "type": "number"
  },
  "DEFAULT_MIN_BATCH_SIZE": {
    "env_var_name": "DEFAULT_MIN_BATCH_SIZE",
    "value": "1",
    "title": "Default Starting Batch Size",
    "description": "Batch size for the first request, which will be multiplied by the growth factor every subsequent request",
    "required": false,
    "type": "number"
  },
  "DEFAULT_BATCH_SIZE_GROWTH_FACTOR": {
    "env_var_name": "DEFAULT_BATCH_SIZE_GROWTH_FACTOR",
    "value": "3",
    "title": "Default Batch Size Growth Factor",
    "description": "Growth factor for dynamic batch size",
    "required": false,
    "type": "number"
  },
  "RAW_OPENAI_OUTPUT": {
    "env_var_name": "RAW_OPENAI_OUTPUT",
    "value": true,
    "title": "Raw OpenAI Output",
    "description": "Raw OpenAI output instead of just the text",
    "required": false,
    "type": "toggle"
  },
  "OPENAI_RESPONSE_ROLE": {
    "env_var_name": "OPENAI_RESPONSE_ROLE",
    "value": "assistant",
    "title": "OpenAI Response Role",
    "description": "Role of the LLM's Response in OpenAI Chat Completions",
    "required": false,
    "type": "text"
  },
  "OPENAI_SERVED_MODEL_NAME_OVERRIDE": {
    "env_var_name": "OPENAI_SERVED_MODEL_NAME_OVERRIDE",
    "value": "",
    "title": "OpenAI Served Model Name Override",
    "description": "Overrides the name of the served model from model repo/path to specified name, which you will then be able to use the value for the `model` parameter when making OpenAI requests",
    "required": false,
    "type": "text"
  },
  "MAX_CONCURRENCY": {
    "env_var_name": "MAX_CONCURRENCY",
    "value": "300",
    "title": "Max Concurrency",
    "description": "Max concurrent requests per worker. vLLM has an internal queue, so you don't have to worry about limiting by VRAM, this is for improving scaling/load balancing efficiency",
    "required": false,
    "type": "number"
  },
  "MODEL_REVISION": {
    "env_var_name": "MODEL_REVISION",
    "value": "",
    "title": "Model Revision",
    "description": "Model revision (branch) to load",
    "required": false,
    "type": "text"
  },
  "BASE_PATH": {
    "env_var_name": "BASE_PATH",
    "value": "/runpod-volume",
    "title": "Base Path",
    "description": "Storage directory for Huggingface cache and model",
    "required": false,
    "type": "text"
  },
  "DISABLE_LOG_REQUESTS": {
    "env_var_name": "DISABLE_LOG_REQUESTS",
    "value": true,
    "title": "Disable Log Requests",
    "description": "Enables or disables vLLM request logging",
    "required": false,
    "type": "toggle"
  },
  "ENABLE_AUTO_TOOL_CHOICE": {
    "env_var_name": "ENABLE_AUTO_TOOL_CHOICE",
    "value": false,
    "title": "Enable Auto Tool Choice",
    "description": "Enables or disables auto tool choice",
    "required": false,
    "type": "toggle"
  },
  "TOOL_CALL_PARSER": {
    "env_var_name": "TOOL_CALL_PARSER",
    "value": "",
    "title": "Tool Call Parser",
    "description": "Tool call parser",
    "required": false,
    "type": "select",
    "options": [
      { "value": "", "label": "None" },
      { "value": "hermes", "label": "Hermes" },
      { "value": "mistral", "label": "Mistral" },
      { "value": "llama3_json", "label": "Llama3 JSON" },
      { "value": "pythonic", "label": "Pythonic" },
      { "value": "internlm", "label": "InternLM" }
    ]
  }
}
}