diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 018508e..e0da84f 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -32,6 +32,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Determine Dockerfile + id: dockerfile + run: | + if [[ "${{ matrix.python-version }}" == "3.10" || "${{ matrix.python-version }}" == "3.11" ]]; then + echo "path=Dockerfile.al2" >> $GITHUB_OUTPUT + else + echo "path=Dockerfile.al2023" >> $GITHUB_OUTPUT + fi + - name: Log in to Container Registry if: github.event_name != 'pull_request' uses: docker/login-action@v3 @@ -57,6 +66,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . + file: ${{ steps.dockerfile.outputs.path }} platforms: linux/amd64 build-args: | PYTHON_VERSION=${{ matrix.python-version }} @@ -82,6 +92,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Determine Dockerfile + id: dockerfile + run: | + if [[ "${{ matrix.python-version }}" == "3.10" || "${{ matrix.python-version }}" == "3.11" ]]; then + echo "path=Dockerfile.al2" >> $GITHUB_OUTPUT + else + echo "path=Dockerfile.al2023" >> $GITHUB_OUTPUT + fi + - name: Log in to Container Registry if: github.event_name != 'pull_request' uses: docker/login-action@v3 @@ -107,6 +126,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . + file: ${{ steps.dockerfile.outputs.path }} platforms: linux/arm64 build-args: | PYTHON_VERSION=${{ matrix.python-version }} diff --git a/Dockerfile.al2 b/Dockerfile.al2 new file mode 100644 index 0000000..d3b7681 --- /dev/null +++ b/Dockerfile.al2 @@ -0,0 +1,34 @@ +ARG PYTHON_VERSION=3.11 +FROM amazonlinux:2 + +# Install base dependencies +RUN yum -y install git zip tar gzip gcc gcc-c++ cmake make \ + wget openssl11-devel bzip2-devel libffi-devel \ + zlib-devel xz-devel sqlite-devel readline-devel xz && \ + yum clean all + +# Accept Python version as build arg (supports 3.10, 3.11) +ARG PYTHON_VERSION + +# Install Python - build from source for AL2 +RUN cd /tmp && \ + wget https://www.python.org/ftp/python/${PYTHON_VERSION}.0/Python-${PYTHON_VERSION}.0.tar.xz && \ + tar xf Python-${PYTHON_VERSION}.0.tar.xz && \ + cd Python-${PYTHON_VERSION}.0 && \ + ./configure --enable-optimizations --with-ensurepip=install && \ + make -j$(nproc) && \ + make altinstall && \ + cd / && \ + rm -rf /tmp/Python-${PYTHON_VERSION}.0* + +# Create symlink for easier access +RUN ln -sf /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python3 && \ + ln -sf /usr/local/bin/pip${PYTHON_VERSION} /usr/local/bin/pip3 + +# Upgrade pip +RUN python3 -m pip install --upgrade pip + +ADD package.sh / +RUN chmod +x /package.sh + +ENTRYPOINT ["/package.sh"] diff --git a/Dockerfile.al2023 b/Dockerfile.al2023 new file mode 100644 index 0000000..b2be5ab --- /dev/null +++ b/Dockerfile.al2023 @@ -0,0 +1,47 @@ +ARG PYTHON_VERSION=3.13 +FROM amazonlinux:2023 + +# Install base dependencies +RUN dnf -y install git zip tar gzip gcc gcc-c++ cmake \ + wget openssl-devel bzip2-devel libffi-devel \ + zlib-devel xz-devel sqlite-devel readline-devel xz && \ + dnf clean all + +# Accept Python version as build arg (supports 3.10, 3.11, 3.12, 3.13, 3.14) +ARG PYTHON_VERSION + +# Install Python - either from dnf or build from source +RUN if dnf list available python${PYTHON_VERSION} 2>/dev/null; then \ + echo "Installing Python ${PYTHON_VERSION} from dnf..."; \ + dnf -y install python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ + dnf clean all; \ + else \ + echo "Building Python ${PYTHON_VERSION} from source..."; \ + cd /tmp && \ + wget https://www.python.org/ftp/python/${PYTHON_VERSION}.0/Python-${PYTHON_VERSION}.0.tar.xz && \ + tar xf Python-${PYTHON_VERSION}.0.tar.xz && \ + cd Python-${PYTHON_VERSION}.0 && \ + ./configure --enable-optimizations --with-ensurepip=install && \ + make -j$(nproc) && \ + make altinstall && \ + cd / && \ + rm -rf /tmp/Python-${PYTHON_VERSION}.0*; \ + fi + +# Create symlink for easier access +RUN if [ -f /usr/local/bin/python${PYTHON_VERSION} ]; then \ + ln -sf /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python3; \ + ln -sf /usr/local/bin/pip${PYTHON_VERSION} /usr/local/bin/pip3; \ + elif [ -f /usr/bin/python${PYTHON_VERSION} ]; then \ + ln -sf /usr/bin/python${PYTHON_VERSION} /usr/local/bin/python3; \ + ln -sf /usr/bin/pip${PYTHON_VERSION} /usr/local/bin/pip3; \ + fi + +# Upgrade pip (skip if pip not available as module) +RUN python3 -m pip install --upgrade pip 2>/dev/null || \ + python3 -m ensurepip && python3 -m pip install --upgrade pip + +ADD package.sh / +RUN chmod +x /package.sh + +ENTRYPOINT ["/package.sh"] \ No newline at end of file diff --git a/readme.md b/readme.md index e1791f2..5c34aee 100644 --- a/readme.md +++ b/readme.md @@ -30,9 +30,12 @@ Simply run the build script with no arguments: - Reads from `requirements.txt` in the current directory - Builds for both x86_64 and arm64 architectures - Creates single combined archive per architecture +- Automatically uses the correct Amazon Linux version for Lambda compatibility **Result**: `combined-python3.14-x86_64.zip` and `combined-python3.14-aarch64.zip` in `./output/` +> **Note**: This tool uses Amazon Linux 2 for Python 3.10-3.11 and Amazon Linux 2023 for Python 3.12+ to ensure compatibility with [AWS Lambda Python runtimes](https://docs.aws.amazon.com/lambda/latest/dg/lambda-python.html). + ### Customizing the Build ```bash @@ -131,15 +134,21 @@ docker run --rm \ ## Supported Python Versions -| Python Version | Amazon Linux 2023 | Installation Method | Status | -|----------------|-------------------|---------------------|--------| -| 3.10 | ✅ Available | dnf package | Supported | -| 3.11 | ✅ Available | dnf package | Supported | -| 3.12 | ✅ Available | dnf package | Supported | -| 3.13 | ✅ Available | dnf package | Supported | -| 3.14 | ⚙️ Built from source | Compiled with optimizations | Supported (Default) | +This project automatically uses the appropriate Amazon Linux version based on [AWS Lambda Python runtime requirements](https://docs.aws.amazon.com/lambda/latest/dg/lambda-python.html): + +| Python Version | Base OS | Installation Method | Lambda Runtime Compatibility | +|----------------|---------|---------------------|------------------------------| +| 3.10 | Amazon Linux 2 | Built from source | ✅ python3.10 runtime | +| 3.11 | Amazon Linux 2 | Built from source | ✅ python3.11 runtime | +| 3.12 | Amazon Linux 2023 | dnf package | ✅ python3.12 runtime | +| 3.13 | Amazon Linux 2023 | dnf package | ✅ python3.13 runtime | +| 3.14 | Amazon Linux 2023 | Built from source | ⚙️ Future runtime (Default) | -**Note**: Python 3.14 is automatically compiled from source when building the Docker image, which takes longer but ensures compatibility with the latest Python version. +**Key Points:** +- **Python 3.10 & 3.11**: Use Amazon Linux 2 (matches Lambda runtime environment) +- **Python 3.12 & 3.13**: Use Amazon Linux 2023 (matches Lambda runtime environment) +- **Python 3.14**: Uses Amazon Linux 2023, compiled from source for latest features +- All versions are built with development headers for compiling binary packages ## Output Formats @@ -235,7 +244,9 @@ Build for Python 3.10 with specific requirements: ``` . -├── Dockerfile # Multi-version, multi-arch capable Docker image +├── Dockerfile.al2 # Amazon Linux 2 image (Python 3.10-3.11) +├── Dockerfile.al2023 # Amazon Linux 2023 image (Python 3.12-3.14) +├── Dockerfile # Default (points to AL2023) ├── package.sh # Packaging script with requirements.txt support ├── build-multiarch.sh # Helper script for multi-arch builds ├── requirements.example.txt @@ -244,11 +255,12 @@ Build for Python 3.10 with specific requirements: ## How It Works -1. **Base Image**: Uses Amazon Linux 2023 for Lambda runtime compatibility +1. **Base Image Selection**: Automatically chooses Amazon Linux 2 (for Python 3.10-3.11) or Amazon Linux 2023 (for Python 3.12+) to match AWS Lambda runtime environments 2. **Python Installation**: Installs specified Python version (3.10-3.14) with development headers -3. **Package Installation**: Uses pip with `--target` flag to install packages -4. **Packaging**: Creates zip archives with proper Lambda Layer structure -5. **Multi-arch**: Leverages Docker buildx for platform-specific builds +3. **Package Installation**: Uses pip with `--target` flag to install packages with proper dependencies +4. **Packaging**: Creates zip archives with proper Lambda Layer structure (`python/lib/pythonX.Y/site-packages/`) +5. **Multi-arch**: Leverages Docker buildx for platform-specific builds (x86_64 and arm64) +6. **Runtime Compatibility**: Ensures binary compatibility with AWS Lambda execution environments ## Troubleshooting