import TilesGrid from "@site/src/components/TilesGrid"; import TileCard from "@site/src/components/TileCard"; import FeatureHighlights from "@site/src/components/FeatureHighlights"; import { Shield, Globe, Zap, Users, Wrench, Play, GitBranch, BarChart3, Lock, DollarSign, Gauge, ShieldCheck } from "lucide-react"; import GenAIDemoCard from "@site/src/content/genai_demo_card.mdx";

MLflow AI Gateway

MLflow's AI Gateway provides a unified interface for deploying and managing multiple LLM providers within your organization. It simplifies interactions with services like OpenAI, Anthropic, and others through a single, secure endpoint.

The gateway excels in production environments where organizations need to manage multiple LLM providers securely while maintaining operational flexibility. Advanced routing capabilities enable traffic splitting for A/B testing and automatic failover chains for high availability.

MLflow AI Gateway also offers passthrough endpoints, enabling requests to be forwarded in providers' native formats. This feature allows you to access provider-specific capabilities as soon as they become available.

<FeatureHighlights features={[ { icon: Globe, title: "Unified Interface", description: "Access multiple LLM providers through a single endpoint, eliminating the need to integrate with each provider individually." }, { icon: Shield, title: "Centralized Security", description: "Store LLM provider API keys in one secure location with request/response logging for audit trails and compliance." }, { icon: GitBranch, title: "Advanced Routing", description: "Traffic splitting for A/B testing and automatic fallbacks ensure high availability across providers." }, { icon: Zap, title: "Zero-Downtime Updates", description: "Add, remove, or modify endpoints dynamically without restarting the server or disrupting running applications." }, { icon: BarChart3, title: "Cost Optimization", description: "Monitor usage across providers and optimize costs by routing requests to the most efficient models." }, { icon: Users, title: "Team Collaboration", description: "Shared endpoint configurations and standardized access patterns across development teams." } ]} />

<GenAIDemoCard />

Next Steps

<TilesGrid> <TileCard icon={Play} title="Quickstart" description="Get your AI Gateway running in minutes with a simple walkthrough" href="./quickstart" linkText="Get started →" /> <TileCard icon={Shield} title="LLM Connections" description="Create and manage LLM connections to securely store provider API keys" href="./api-keys/create-and-manage" linkText="Manage connections →" /> <TileCard icon={Wrench} title="Endpoints" description="Create, configure, and query AI model endpoints" href="./endpoints/create-and-manage" linkText="Configure endpoints →" /> <TileCard icon={GitBranch} title="Traffic Routing" description="Configure traffic splitting and fallbacks for high availability" href="./traffic-routing-fallbacks" linkText="Learn routing →" /> <TileCard icon={BarChart3} title="Usage Tracking" description="Monitor endpoint usage, performance, token consumption, and costs" href="./usage-tracking" linkText="View metrics →" /> <TileCard icon={DollarSign} title="Budget Alerts & Limits" description="Set spending limits and configure alerts for cost management" href="./budget-alerts-limits" linkText="Manage budgets →" /> <TileCard icon={ShieldCheck} title="Guardrails" description="Enforce content policies with LLM-powered judges that block or sanitize requests and responses" href="./guardrails" linkText="Configure guardrails →" /> <TileCard icon={Lock} title="Authentication" description="Configure HTTP Basic Authentication for AI Gateway resources" href="/self-hosting/security/basic-http-auth#ai-gateway-permissions" linkText="Learn more →" /> <TileCard icon={Gauge} title="Performance & Benchmarks" description="Benchmark results and how to measure gateway overhead in your own environment" href="./benchmarks" linkText="View benchmarks →" /> <TileCard icon={Play} title="LLM Playground" description="Test endpoints interactively from the MLflow UI with multi-turn chat, sampling parameters, tools, and structured output" href="/genai/prompt-registry/playground" linkText="Open Playground →" /> </TilesGrid>