Load Balancing
Manage your workload effectively with Rubeus's custom weight-based distribution across multiple API keys or providers.
# Load balance 50-50 between gpt-3.5-turbo and claude-v1
curl --location 'http://127.0.0.1:8787/complete' \
--header 'Content-Type: application/json' \
--data '{
"config": {
"mode": "loadbalance",
"options": [{
"provider": "openai",
"weight": 0.5,
"params_to_override": { "model": "gpt-3.5-turbo" }
}, {
"provider": "anthropic",
"weight": 0.5,
"params_to_override": { "model": "claude-v1" }
}]
},
"params": {
"messages": {"role": "user","content":"What are the top 10 happiest countries in the world?"},
"max_tokens": 50,
"user": "jbu3470"
}
}'