Troubleshooting: Successful Build but ECS Runtime Failures

# Check what image ECS is actually using
aws ecs describe-task-definition --task-definition your-task-name | grep image

# Verify the image digest
docker inspect your-registry/waiide:latest | grep -A 5 "RepoDigests"

# Compare with what ECS pulled
aws ecs describe-tasks --cluster your-cluster --tasks your-task-id | grep imageDigest

# Verify the pushed image size (should be ~3-4GB)
aws ecr describe-images --repository-name waiide --image-ids imageTag=latest

# Or for Docker Hub
docker manifest inspect calliopeai/waiide:latest

# Check if you accidentally pushed multi-arch manifest
docker buildx imagetools inspect your-registry/waiide:latest

# ECS might be pulling wrong architecture
# Ensure you pushed specifically linux/amd64
docker buildx build --platform linux/amd64 --push -t your-registry/waiide:latest-amd64 .

# Force new task definition revision
aws ecs register-task-definition --cli-input-json file://task-def.json

# Force new deployment
aws ecs update-service --cluster your-cluster --service your-service --force-new-deployment

# In your GitHub Actions workflow, add:
- name: Verify built image
  run: |
    docker run --rm your-registry/waiide:latest uname -m
    docker run --rm your-registry/waiide:latest ls -la /home/calliope/WAIIDE-server/
    docker run --rm your-registry/waiide:latest ls -la /home/calliope/scripts/

# Pull the exact image ECS is using
docker pull your-registry/waiide:latest

# Test it locally
docker run -it --rm your-registry/waiide:latest bash -c "
  echo '=== Checking WAIIDE Server ==='
  ls -la /home/calliope/WAIIDE-server/ | head -10
  echo '=== Checking for server-main.js ==='
  find /home/calliope -name 'server-main.js' 2>/dev/null | head -5
  echo '=== Checking scripts ==='
  ls -la /home/calliope/scripts/ | head -10
  echo '=== Image architecture ==='
  uname -m
"

# Check if the image has the expected layers
docker history your-registry/waiide:latest | grep -E "(WAIIDE|scripts|COPY)"

# Enable ECS Exec
aws ecs update-service --cluster your-cluster --service your-service --enable-execute-command

# Exec into running container
aws ecs execute-command --cluster your-cluster --task your-task-id --container waiide --interactive --command "/bin/bash"

# Inside container, check:
find / -name "server-main.js" 2>/dev/null
find / -name "jupyter_server_config.py" 2>/dev/null
ls -la /home/

{
  "image": "your-registry/waiide@sha256:abc123...",
  "taskDefinitionArn": "..."
}

# In GitHub Actions
docker buildx build \
  --platform linux/amd64 \
  --push \
  -t your-registry/waiide:$(git rev-parse --short HEAD) \
  -t your-registry/waiide:latest \
  .

# Update ECS to use specific tag
"image": "your-registry/waiide:a1b2c3d"

# Add to GitHub Actions after push
- name: Verify pushed image
  run: |
    docker pull ${{ env.REGISTRY }}/waiide:latest
    docker run --rm ${{ env.REGISTRY }}/waiide:latest ls -la /home/calliope/WAIIDE-server/

- name: Build and verify
  run: |
    docker buildx build --platform linux/amd64 --load -t waiide:test .
    
    # Verify critical files before pushing
    docker run --rm waiide:test bash -c "
      test -f /home/calliope/WAIIDE-server/server-main.js || test -f /home/calliope/WAIIDE-server/out/server-main.js || exit 1
      test -f /home/calliope/scripts/jupyter_server_config.py || exit 1
      echo 'Build verification passed!'
    "
    
    # Only push if verification passed
    docker buildx build --platform linux/amd64 --push -t ${{ env.REGISTRY }}/waiide:latest .