NVIDIA NIM

export NGC_API_KEY=nvapi_your_key_here

docker run --gpus all \
  -e NGC_API_KEY=$NGC_API_KEY \
  -p 8000:8000 \
  nvcr.io/nim/meta/llama-3.1-8b-instruct:latest

# Cognisafe proxy env vars
UPSTREAM_URL=http://nim-host.internal:8000

import cognisafe
from openai import OpenAI

cognisafe.configure(
    api_key="csk_your_key_here",
    project_id="my-app",
    proxy_url="http://cognisafe-proxy:8080",
)
cognisafe.patch_openai()

client = OpenAI(
    api_key="nvapi_your_key_here",  # forwarded to NIM by the proxy
    base_url="http://cognisafe-proxy:8080/v1",
)

response = client.chat.completions.create(
    model="meta/llama-3.1-8b-instruct",
    messages=[{"role": "user", "content": "Explain quantum entanglement."}],
)

UPSTREAM_URL=https://integrate.api.nvidia.com

client = OpenAI(
    api_key="nvapi_your_key_here",
    base_url="http://cognisafe-proxy:8080/v1",
)

response = client.chat.completions.create(
    model="nvidia/llama-3.1-nemotron-70b-instruct",
    messages=[{"role": "user", "content": "Hello"}],
)

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nim-llama
  namespace: cognisafe-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nim-llama
  template:
    metadata:
      labels:
        app: nim-llama
    spec:
      runtimeClassName: nvidia
      containers:
        - name: nim
          image: nvcr.io/nim/meta/llama-3.1-8b-instruct:latest
          ports:
            - containerPort: 8000
          env:
            - name: NGC_API_KEY
              valueFrom:
                secretKeyRef:
                  name: ngc-secret
                  key: api-key
          resources:
            limits:
              nvidia.com/gpu: "1"
              memory: "24Gi"
            requests:
              nvidia.com/gpu: "1"
          readinessProbe:
            httpGet:
              path: /v1/health/ready
              port: 8000
            initialDelaySeconds: 60
            periodSeconds: 10
---
apiVersion: v1
kind: Service
metadata:
  name: nim-llama
  namespace: cognisafe-system
spec:
  selector:
    app: nim-llama
  ports:
    - port: 8000
      targetPort: 8000

docker run --gpus all \
  -e NGC_API_KEY=$NGC_API_KEY \
  -p 8001:8000 \
  nvcr.io/nim/nvidia/llama-3.1-nemo-guardrails:latest

OPENAI_API_KEY=nvapi_your_key_here
SCORER_MODEL=nvidia/llama-3.1-nemo-guardrails
OPENAI_BASE_URL=http://nim-guardrails-host.internal:8001/v1