Hugging Face (TGI & Inference Endpoints)

docker run --gpus all -p 8080:80 \
  -v $HOME/.cache/huggingface:/data \
  ghcr.io/huggingface/text-generation-inference:latest \
  --model-id meta-llama/Llama-3.1-8B-Instruct

# Cognisafe proxy env vars
UPSTREAM_URL=http://tgi-host.internal:8080

import cognisafe
from openai import OpenAI

cognisafe.configure(
    api_key="csk_your_key_here",
    project_id="my-app",
    proxy_url="http://localhost:8080",  # Cognisafe proxy
)
cognisafe.patch_openai()

client = OpenAI(api_key="hf-token-not-needed-for-local-tgi")

response = client.chat.completions.create(
    model="tgi",  # TGI ignores the model name — it serves whatever was loaded
    messages=[{"role": "user", "content": "Summarise this document..."}],
)

UPSTREAM_URL=https://xyz.us-east-1.aws.endpoints.huggingface.cloud

import cognisafe
from openai import OpenAI

cognisafe.configure(
    api_key="csk_your_key_here",
    project_id="prod",
    proxy_url="http://your-cognisafe-proxy:8080",
)
cognisafe.patch_openai()

client = OpenAI(api_key="hf_your_token_here")  # HF token forwarded by proxy

response = client.chat.completions.create(
    model="tgi",
    messages=[{"role": "user", "content": "Hello"}],
)

# safety_worker env vars
OPENAI_API_KEY=dummy          # any non-empty value
SCORER_MODEL=tgi              # model name TGI ignores anyway
OPENAI_BASE_URL=http://tgi-scoring-host.internal:8080/v1

apiVersion: apps/v1
kind: Deployment
metadata:
  name: tgi
spec:
  replicas: 1
  selector:
    matchLabels:
      app: tgi
  template:
    spec:
      containers:
        - name: tgi
          image: ghcr.io/huggingface/text-generation-inference:latest
          args:
            - --model-id=meta-llama/Llama-3.1-8B-Instruct
            - --port=8080
          ports:
            - containerPort: 8080
          resources:
            limits:
              nvidia.com/gpu: "1"
          env:
            - name: HUGGING_FACE_HUB_TOKEN
              valueFrom:
                secretKeyRef:
                  name: hf-secret
                  key: token
---
apiVersion: v1
kind: Service
metadata:
  name: tgi
spec:
  selector:
    app: tgi
  ports:
    - port: 8080
      targetPort: 8080