Page History

services:
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm-server
    # Runtime (requires nvidia-container-toolkit)
    runtime: nvidia
    environment:
      - HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN}
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
      - ./models:/models
      - ./logs:/logs
    ports:
      - "127.0.0.1:${VLLM_PORT}:8000"
    command: >
      --model ${MODEL_NAME}
      --tensor-parallel-size ${TENSOR_PARALLEL_SIZE}
      --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION}
      --max-model-len ${MAX_MODEL_LEN}
      --api-key ${VLLM_API_KEY}
      --port 8000
      --host 0.0.0.0
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 300s
    restart: unless-stopped
    networks:
      - vllm-network
    logging:
      driver: "json-file"
      options:
        max-size: "100m"
        max-file: "10"

  nginx:
    image: nginx:alpine
    container_name: nginx-proxy
    ports:
      - "${NGINX_PORT}:80"
      - "${NGINX_SSL_PORT}:443"
    volumes:
      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
      - ./nginx/logs:/var/log/nginx
      - ./nginx/ssl:/etc/nginx/ssl:ro
    depends_on:
      - vllm
    healthcheck:
      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    restart: unless-stopped
    networks:
      - vllm-network
    logging:
      driver: "json-file"
      options:
        max-size: "50m"
        max-file: "5"

networks:
  vllm-network:
    driver: bridge

#!/bin/bash

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"

cd "$PROJECT_DIR"

echo "=========================================="
echo "vLLM Deployment Startup"
echo "=========================================="

# Check if .env file exists
if [ ! -f .env ]; then
    echo "Error: .env file not found!"
    echo "Please create .env file with your configuration"
    exit 1
fi

# Load environment variables
source .env

# Validate required environment variables
if [ -z "$HUGGING_FACE_HUB_TOKEN" ]; then
    echo "Error: HUGGING_FACE_HUB_TOKEN not set in .env!"
    exit 1
fi

if [ -z "$VLLM_API_KEY" ]; then
    echo "Warning: VLLM_API_KEY not set. Using default (not secure!)"
fi

# Check GPU availability
echo " Checking GPU availability..."
if ! nvidia-smi &> /dev/null; then
    echo "Error: NVIDIA GPU not detected!"
    echo "Please ensure NVIDIA drivers are installed"
    exit 1
fi

echo "GPU detected:"
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader

# Check Docker
echo "Checking Docker..."
if ! docker --version &> /dev/null; then
    echo "Error: Docker not installed!"
    exit 1
fi

# Check Docker Compose
if ! docker compose version &> /dev/null; then
    echo "Error: Docker Compose not installed!"
    exit 1
fi

# Create necessary directories
echo "Creating directories..."
mkdir -p nginx/logs
mkdir -p logs
mkdir -p models

# Stop existing containers if running
echo "Stopping existing containers (if any)..."
docker compose down 2>/dev/null || true

# Pull latest images
echo "Pulling Docker images..."
docker compose pull

# Start services
echo "Starting services..."
docker compose up -d

# Wait for services to be ready
echo "Waiting for services to start (this may take a few minutes for model loading)..."
sleep 10

# Check startup logs
echo "Checking startup logs..."
echo "--- vLLM logs ---"
docker compose logs --tail=20 vllm

echo ""
echo "--- Nginx logs ---"
docker compose logs --tail=10 nginx

# Wait more for model loading
echo "Waiting for model to load completely..."
MAX_WAIT=300  # 5 minutes maximum
WAITED=0

while [ $WAITED -lt $MAX_WAIT ]; do
    if curl -s -f -H "Authorization: Bearer ${VLLM_API_KEY}" http://localhost:8000/health > /dev/null 2>&1; then
        echo "vLLM is ready!"
        break
    fi
    echo -n "."
    sleep 5
    WAITED=$((WAITED + 5))
done

if [ $WAITED -ge $MAX_WAIT ]; then
    echo ""
    echo "Warning: vLLM health check timeout. Check logs with: docker compose logs vllm"
fi

# Run health check
echo ""
echo "Running health check..."
python3 scripts/health_check.py

echo ""
echo "=========================================="
echo " Deployment Complete!"
echo "=========================================="
echo ""
echo " Access Points:"
echo "  - vLLM API: http://${SERVER_IP}"
echo "  - API Docs: http://${SERVER_IP}/docs"
echo ""
echo " Useful Commands:"
echo "  - View logs:        docker compose logs -f [vllm|nginx]"
echo "  - Health check:     python3 scripts/health_check.py"
echo "  - Start monitoring: python3 scripts/monitor.py"
echo "  - Stop services:    docker compose down"
echo "  - Restart services: docker compose restart"
echo ""
echo " API Usage:"
echo "  curl -X POST http://${SERVER_IP}/v1/chat/completions \\"
echo "    -H \"Content-Type: application/json\" \\"
echo "    -H \"Authorization: Bearer ${VLLM_API_KEY}\" \\"
echo "    -d '{\"model\": \"${MODEL_NAME}\", \"messages\": [{\"role\": \"user\", \"content\": \"Hello!\"}]}'"
echo ""