Add support for jenkins as CI provider

[cojmeister]

Hey,
I love this tool!
I want to add support for Jenkins as a CI provider.
I can do it, but I need some help/mentoring in order to properly define a CI.
From what I see in the current CI is:

• Checkout
• Setup python
• Build for archs
• Stash
  For each of the three OSs (windows, macos, linux). And then:
• Unstash
• Publish to pypi

I also see that there is extensive use of maturin-action - which isn't one to one with maturin cli tool.

As a first draft I've created the following workflow. It first checks out. Then sets up python and builds parallel for each agent (windows ubuntu and macos), and then stashes the result. Finally it releases a version to pypi.

pipeline {
    agent none
    stages {
        stage('Checkout') {
            agent any
            steps {
                checkout scm
            }
        }
        stage('Build') {
            failFast true
            parallel {
                stage('windows') {
                    agent {
                        label "windows-latest"
                    }
                    stages {
                        stage('Setup maturin') {
                            steps {
                                sh 'pip install maturin'
                            }
                        }
                        stage('Build and stash') {
                            matrix {
                                axes {
                                    axis {
                                        name 'ARCH'
                                        values 'x64', 'x86'
                                    }
                                }
                                stages {
                                    stage('build') {
                                        sh "maturin build --release --out dist --find-interpreter --target $ARCH-pc-windows-gnu"
                                    }
                                }
                            }
                        }
                       stage('stash') {
                            stash includes: 'dist/**/*', name: 'windows'
                        }
                    }
                }
                stage('linux') {
                    agent {
                        label "ubuntu-latest"
                    }
                    stages {
                        stage('Setup maturin') {
                            steps {
                                sh 'pip3.10 install maturin'
                            }
                        }
                        stage('Build and stash') {
                            matrix {
                                axes {
                                    axis {
                                        name 'ARCH'
                                        values 'x86_64', 'x86', 'aarch64', 'armv7', 's390x', 'ppc64le'
                                    }
                                }
                                stages {
                                    stage('build') {
                                        // This won't work for all given archs - need to understand how to run with maturin
                                        sh "maturin build --release --out dist --find-interpreter --target $ARCH-unknown-linux-gnu"
                                        // Need to add sccache and manylinux - currently getting error on sccache (locally)
                                    }
                                }
                            }
                        }
                       stage('stash') {
                            stash includes: 'dist/**/*', name: 'ubuntu'
                        }
                    }
                }
                stage('macos') {
                    agent {
                        label "macos-latest"
                    }
                    stages {
                        stage('Setup maturin') {
                            steps {
                                sh 'pip3.10 install maturin'
                            }
                        }
                        stage('Build') {
                            matrix {
                                axes {
                                    axis {
                                        name 'ARCH'
                                        values 'x86_64', 'aarch64'
                                    }
                                }
                                stages {
                                    stage('build') {
                                        sh "maturin build --release --out dist --find-interpreter --target $ARCH-apple-darwin"
                                    }
                                }
                            }
                        }
                       stage('stash') {
                            stash includes: 'dist/**/*', name: 'macos'
                        }
                    }
                }
                stage('sdist') {
                    agent {
                        label 'ubuntu-latest'
                    }
                    stages {
                        stage('build sdist') {
                            // For some reason this is causing an incredibly large file (locally)
                            sh 'maturin build --sdist  --out dist' 
                        }
                        stage('stash sdist') {
                            stash includes: 'dist/**/*', name: 'sdist'
                        }
                    }
                }
            }
        }
        stage('Release') {
            when {} // Need to better understand when to release
            agent {
                label "ubuntu-latest"
            }
            environment { 
                MATURIN_PYPI_TOKEN = credentials('my-pypi-api-token') 
            }
            steps {
                unstash 'windows'
                unstash 'macos'
                unstash 'linux'
                unstash 'sdist'
                // Need to deal with Pypi token
                sh 'maturin upload --non-interactive --skip-existing ./dist/*'
            }
        }
    }
}

What do you think?
Is this build good? Should I add our remove anything?

[messense]

Looks reasonable, but manylinux is a must have feature.

[cojmeister]

@messense can you explain what the feature does?
I'm not understanding it from the documentation, nor from the code

[messense]

@cojmeister https://github.com/pypa/manylinux

[cojmeister]

@messense ok
What I don't understand is the logic in the maturin-action. I mean, shouldn't the action replicate the build command in the CLI? In any case, when I set manylinux, I'm basically accessing this HashMap?

const DEFAULT_CONTAINERS: Record<string, Record<string, string>> = {
  'x86_64-unknown-linux-gnu': {
    auto: 'quay.io/pypa/manylinux2014_x86_64:latest',
    '2010': 'quay.io/pypa/manylinux2010_x86_64:latest',
    '2_12': 'quay.io/pypa/manylinux2010_x86_64:latest',
    '2014': 'quay.io/pypa/manylinux2014_x86_64:latest',
    '2_17': 'quay.io/pypa/manylinux2014_x86_64:latest',
    '2_24': 'quay.io/pypa/manylinux_2_24_x86_64:latest',
    '2_28': 'quay.io/pypa/manylinux_2_28_x86_64:latest'
  },
  'x86_64-unknown-linux-musl': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:x86_64-musl',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:x86_64-musl',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:x86_64-musl'
  },
  'i686-unknown-linux-gnu': {
    auto: 'quay.io/pypa/manylinux2014_i686:latest',
    '2010': 'quay.io/pypa/manylinux2010_i686:latest',
    '2_12': 'quay.io/pypa/manylinux2010_i686:latest',
    '2014': 'quay.io/pypa/manylinux2014_i686:latest',
    '2_17': 'quay.io/pypa/manylinux2014_i686:latest',
    '2_24': 'quay.io/pypa/manylinux_2_24_i686:latest'
  },
  'i686-unknown-linux-musl': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:i686-musl',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:i686-musl',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:i686-musl'
  },
  'aarch64-unknown-linux-gnu': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:aarch64',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:aarch64',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:aarch64',
    '2_24': 'messense/manylinux_2_24-cross:aarch64',
    '2_28': 'ghcr.io/rust-cross/manylinux_2_28-cross:aarch64'
  },
  'aarch64-unknown-linux-musl': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:aarch64-musl',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:aarch64-musl',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:aarch64-musl'
  },
  'armv7-unknown-linux-gnueabihf': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:armv7',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:armv7',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:armv7',
    '2_24': 'messense/manylinux_2_24-cross:armv7',
    '2_28': 'ghcr.io/rust-cross/manylinux_2_28-cross:armv7'
  },
  'armv7-unknown-linux-musleabihf': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:armv7-musleabihf',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:armv7-musleabihf',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:armv7-musleabihf'
  },
  'powerpc64-unknown-linux-gnu': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:ppc64',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:ppc64',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:ppc64'
  },
  'powerpc64le-unknown-linux-gnu': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:ppc64le',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:ppc64le',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:ppc64le',
    '2_24': 'messense/manylinux_2_24-cross:ppc64le',
    '2_28': 'ghcr.io/rust-cross/manylinux_2_28-cross:ppc64le'
  },
  'powerpc64le-unknown-linux-musl': {
    auto: 'ghcr.io/rust-cross/rust-musl-cross:powerpc64le-musl',
    musllinux_1_1: 'ghcr.io/rust-cross/rust-musl-cross:powerpc64le-musl',
    musllinux_1_2: 'ghcr.io/rust-cross/rust-musl-cross:powerpc64le-musl'
  },
  's390x-unknown-linux-gnu': {
    auto: 'ghcr.io/rust-cross/manylinux2014-cross:s390x',
    '2014': 'ghcr.io/rust-cross/manylinux2014-cross:s390x',
    '2_17': 'ghcr.io/rust-cross/manylinux2014-cross:s390x',
    '2_24': 'messense/manylinux_2_24-cross:s390x',
    '2_28': 'ghcr.io/rust-cross/manylinux_2_28-cross:s390x'
  }
}

I'd like to think that copying this as is is not a smart idea, any better recommendations?

[messense]

I mean, shouldn't the action replicate the build command in the CLI?

Yes but the build command needs to run in a manylinux compatible environment, hence the docker containers.