>>> from huggingface_hub import list_inference_catalog
>>> catalog = list_inference_catalog(task="text-generation", accelerator="neuron")
>>> catalog[0]
InferenceCatalogModel(repo_id='meta-llama/Llama-3.1-8B-Instruct', name='Llama-3.1-8B-Instruct', ...)
>>> catalog[0].recipes
[InferenceCatalogRecipe(id='sizzling-biryani-g4xsi1ac', accelerator='gpu', engine='vllm', gguf_file=None, revision=None),
 InferenceCatalogRecipe(id='artisanal-quinoa-yz9ynamx', accelerator='neuron', engine='vllmNeuron', gguf_file=None, revision=None)]

>>> from huggingface_hub import create_inference_endpoint_from_catalog
>>> endpoint = create_inference_endpoint_from_catalog(repo_id="openai/gpt-oss-120b")
>>> endpoint = create_inference_endpoint_from_catalog(recipe_id="artisanal-quinoa-yz9ynamx")

>>> from huggingface_hub import list_inference_endpoints_hardware

>>> for hw in list_inference_endpoints_hardware():
...     if hw.accelerator == "gpu" and hw.status == "available":
...         print(hw.vendor, hw.region, hw.instance_type, hw.instance_size, hw.price_per_hour)
aws us-east-1 nvidia-l4 x1 0.8
aws us-east-1 nvidia-l4 x4 3.8
...

>>> list_inference_endpoints_hardware(namespace="my-org")

hf endpoints hardware --accelerator gpu

>>> from huggingface_hub import create_inference_endpoint

>>> endpoint = create_inference_endpoint(
...     "my-endpoint-name",
...     repository="gpt2",
...     framework="pytorch",
...     task="text-generation",
...     accelerator="cpu",
...     vendor="aws",
...     region="us-east-1",
...     type="authenticated",
...     instance_size="x2",
...     instance_type="intel-icl"
... )

hf endpoints deploy my-endpoint-name --repo gpt2 --framework pytorch --accelerator cpu --vendor aws --region us-east-1 --instance-size x2 --instance-type intel-icl --task text-generation

# Deploy from the catalog with a single command
hf endpoints catalog deploy --repo openai/gpt-oss-120b

# Deploy from the catalog with a specific accelerator
hf endpoints catalog deploy --repo openai/gpt-oss-120b --accelerator gpu

# Deploy an exact catalog recipe, as listed by 'hf endpoints catalog ls'
hf endpoints catalog deploy --recipe artisanal-quinoa-yz9ynamx

>>> endpoint
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='pending', url=None)

hf endpoints describe my-endpoint-name

# Start an Inference Endpoint running Zephyr-7b-beta on TGI
>>> from huggingface_hub import create_inference_endpoint
>>> endpoint = create_inference_endpoint(
...     "aws-zephyr-7b-beta-0486",
...     repository="HuggingFaceH4/zephyr-7b-beta",
...     framework="pytorch",
...     task="text-generation",
...     accelerator="gpu",
...     vendor="aws",
...     region="us-east-1",
...     type="authenticated",
...     instance_size="x1",
...     instance_type="nvidia-a10g",
...     custom_image={
...         "healthRoute": "/health",
...         "env": {
...             "MAX_BATCH_PREFILL_TOKENS": "2048",
...             "MAX_INPUT_LENGTH": "1024",
...             "MAX_TOTAL_TOKENS": "1512",
...             "MODEL_ID": "/repository"
...         },
...         "url": "ghcr.io/huggingface/text-generation-inference:1.1.0",
...     },
... )

>>> endpoint = create_inference_endpoint(
...     # Other endpoint and hardware arguments omitted for brevity.
...     custom_image={"url": "private.registry.example/my-image:latest", "port": 8080},
...     container_registry_username="user",
...     container_registry_password="password",
... )

hf endpoints deploy my-endpoint ... --custom-image private.registry.example/my-image:latest \
  --container-registry-username user --container-registry-password password

# Start an Inference Endpoint running the vLLM engine image
>>> endpoint = create_inference_endpoint(
...     "llama-3-1-8b-instruct-vllm",
...     repository="meta-llama/Llama-3.1-8B-Instruct",
...     framework="pytorch",
...     task="text-generation",
...     accelerator="gpu",
...     vendor="aws",
...     region="us-east-1",
...     instance_size="x1",
...     instance_type="nvidia-l4",
...     custom_image={
...         "vLLM": {
...             "url": "vllm/vllm-openai:v0.23.0",
...             "healthRoute": "/health",
...             "port": 8000,
...             "tensorParallelSize": 1,
...         }
...     },
... )

>>> endpoint = create_inference_endpoint(
...     "gpt-oss-120b-vllm",
...     repository="openai/gpt-oss-120b",
...     framework="custom",
...     accelerator="gpu",
...     instance_size="x8",
...     instance_type="nvidia-h200",
...     region="us-east-1",
...     vendor="aws",
...     custom_image={"vLLM": {"url": "vllm/vllm-openai:v0.23.0", "tensorParallelSize": 8}},
... )

hf endpoints deploy gpt-oss-120b-vllm --repo openai/gpt-oss-120b --framework custom \
  --accelerator gpu --instance-size x8 --instance-type nvidia-h200 --region us-east-1 --vendor aws \
  --engine vllm --custom-image vllm/vllm-openai:v0.23.0 --tensor-parallel-size 8

>>> endpoint.update(tensor_parallel_size=4)

>>> from huggingface_hub import get_inference_endpoint, list_inference_endpoints

# Get one
>>> get_inference_endpoint("my-endpoint-name")
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='pending', url=None)

# List all endpoints from an organization
>>> list_inference_endpoints(namespace="huggingface")
[InferenceEndpoint(name='aws-starchat-beta', namespace='huggingface', repository='HuggingFaceH4/starchat-beta', status='paused', url=None), ...]

# List all endpoints from all organizations the user belongs to
>>> list_inference_endpoints(namespace="*")
[InferenceEndpoint(name='aws-starchat-beta', namespace='huggingface', repository='HuggingFaceH4/starchat-beta', status='paused', url=None), ...]

hf endpoints describe my-endpoint-name
hf endpoints ls --namespace huggingface
hf endpoints ls --namespace '*'

>>> endpoint
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='running', url='https://jpj7k2q4j805b727.us-east-1.aws.endpoints.huggingface.cloud')

>>> endpoint.fetch()
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='pending', url=None)

hf endpoints describe my-endpoint-name

# Pending endpoint
>>> endpoint
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='pending', url=None)

# Wait 10s => raises a InferenceEndpointTimeoutError
>>> endpoint.wait(timeout=10)
    raise InferenceEndpointTimeoutError("Timeout while waiting for Inference Endpoint to be deployed.")
huggingface_hub._inference_endpoints.InferenceEndpointTimeoutError: Timeout while waiting for Inference Endpoint to be deployed.

# Wait more
>>> endpoint.wait()
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='running', url='https://jpj7k2q4j805b727.us-east-1.aws.endpoints.huggingface.cloud')

# Run text_generation task:
>>> endpoint.client.text_generation("I am")
' not a fan of the idea of a "big-budget" movie. I think it\'s a'

# Or in an asyncio context:
>>> await endpoint.async_client.text_generation("I am")

>>> endpoint.client
huggingface_hub._inference_endpoints.InferenceEndpointError: Cannot create a client for this Inference Endpoint as it is not yet deployed. Please wait for the Inference Endpoint to be deployed using `endpoint.wait()` and try again.

# Pause and resume endpoint
>>> endpoint.pause()
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='paused', url=None)
>>> endpoint.resume()
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='pending', url=None)
>>> endpoint.wait().client.text_generation(...)
...

# Scale to zero
>>> endpoint.scale_to_zero()
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2', status='scaledToZero', url='https://jpj7k2q4j805b727.us-east-1.aws.endpoints.huggingface.cloud')
# Endpoint is not 'running' but still has a URL and will restart on first call.

hf endpoints pause my-endpoint-name
hf endpoints resume my-endpoint-name
hf endpoints scale-to-zero my-endpoint-name

# Change target model
>>> endpoint.update(repository="gpt2-large")
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2-large', status='pending', url=None)

# Update number of replicas
>>> endpoint.update(min_replica=2, max_replica=6)
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2-large', status='pending', url=None)

# Update to larger instance
>>> endpoint.update(accelerator="cpu", instance_size="x4", instance_type="intel-icl")
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2-large', status='pending', url=None)

# Update the container arguments, e.g. engine flags on an endpoint running vLLM or SGLang.
# Replaces the current value, it is not appended.
>>> endpoint.update(container_args=["--enable-auto-tool-choice", "--tool-call-parser", "lfm2"])
InferenceEndpoint(name='my-endpoint-name', namespace='Wauplin', repository='gpt2-large', status='pending', url=None)

hf endpoints update my-endpoint-name --repo gpt2-large
hf endpoints update my-endpoint-name --min-replica 2 --max-replica 6
hf endpoints update my-endpoint-name --accelerator cpu --instance-size x4 --instance-type intel-icl
hf endpoints update my-endpoint-name --container-args "--enable-auto-tool-choice --tool-call-parser lfm2"
# Merged into the image currently configured on the endpoint.
hf endpoints update my-endpoint-name --tensor-parallel-size 8
# Or replace the image entirely, in which case pass back the settings you want to keep.
hf endpoints update my-endpoint-name --engine vllm --custom-image vllm/vllm-openai:v0.23.0 --port 8000

>>> import asyncio
>>> from huggingface_hub import create_inference_endpoint

# Start endpoint + wait until initialized
>>> endpoint = create_inference_endpoint(name="batch-endpoint",...).wait()

# Run inference
>>> client = endpoint.client
>>> results = [client.text_generation(...) for job in jobs]

# Or with asyncio
>>> async_client = endpoint.async_client
>>> results = await asyncio.gather(*[async_client.text_generation(...) for job in jobs])

# Pause endpoint
>>> endpoint.pause()

>>> import asyncio
>>> from huggingface_hub import get_inference_endpoint

# Get endpoint + wait until initialized
>>> endpoint = get_inference_endpoint("batch-endpoint").resume().wait()

# Run inference
>>> async_client = endpoint.async_client
>>> results = await asyncio.gather(*[async_client.text_generation(...) for job in jobs])

# Pause endpoint
>>> endpoint.pause()