diff --git a/.github/actions/build-dev-builder-image/action.yml b/.github/actions/build-dev-builder-image/action.yml new file mode 100644 index 0000000000..4453d4b16d --- /dev/null +++ b/.github/actions/build-dev-builder-image/action.yml @@ -0,0 +1,70 @@ +name: Build and push dev-builder image +description: Build and push dev-builder image to DockerHub and ACR +inputs: + dockerhub-image-registry: + description: The dockerhub image registry to store the images + required: false + default: docker.io + dockerhub-image-registry-username: + description: The dockerhub username to login to the image registry + required: true + dockerhub-image-registry-password: + description: The dockerhub password to login to the image registry + required: true + dockerhub-image-namespace: + description: The dockerhub namespace of the image registry to store the images + required: false + default: greptime + acr-image-registry: + description: The ACR image registry to store the images + required: true + acr-image-registry-username: + description: The ACR username to login to the image registry + required: true + acr-image-registry-password: + description: The ACR password to login to the image registry + required: true + acr-image-namespace: + description: The ACR namespace of the image registry to store the images + required: false + default: greptime + version: + description: Version of the dev-builder + required: false + default: latest +runs: + using: composite + steps: + - name: Login to Dockerhub + uses: docker/login-action@v2 + with: + registry: ${{ inputs.dockerhub-image-registry }} + username: ${{ inputs.dockerhub-image-registry-username }} + password: ${{ inputs.dockerhub-image-registry-password }} + + - name: Build and push dev builder image to dockerhub + shell: bash + run: + make dev-builder \ + BUILDX_MULTI_PLATFORM_BUILD=true \ + IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \ + IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \ + IMAGE_TAG=${{ inputs.version }} + + - name: Login to ACR + uses: docker/login-action@v2 + continue-on-error: true + with: + registry: ${{ inputs.acr-image-registry }} + username: ${{ inputs.acr-image-registry-username }} + password: ${{ inputs.acr-image-registry-password }} + + - name: Build and push dev builder image to ACR + shell: bash + continue-on-error: true + run: # buildx will cache the images that already built, so it will not take long time to build the images again. + make dev-builder \ + BUILDX_MULTI_PLATFORM_BUILD=true \ + IMAGE_REGISTRY=${{ inputs.acr-image-registry }} \ + IMAGE_NAMESPACE=${{ inputs.acr-image-namespace }} \ + IMAGE_TAG=${{ inputs.version }} diff --git a/.github/actions/build-greptime-binary/action.yml b/.github/actions/build-greptime-binary/action.yml new file mode 100644 index 0000000000..a817111227 --- /dev/null +++ b/.github/actions/build-greptime-binary/action.yml @@ -0,0 +1,51 @@ +name: Build greptime binary +description: Build and upload the single linux artifact +inputs: + base-image: + description: Base image to build greptime + required: true + features: + description: Cargo features to build + required: true + cargo-profile: + description: Cargo profile to build + required: true + artifacts-dir: + description: Directory to store artifacts + required: true + version: + description: Version of the artifact + required: true + release-to-s3-bucket: + description: S3 bucket to store released artifacts + required: true + aws-access-key-id: + description: AWS access key id + required: true + aws-secret-access-key: + description: AWS secret access key + required: true + aws-region: + description: AWS region + required: true +runs: + using: composite + steps: + - name: Build greptime binary + shell: bash + run: | + make build-greptime-by-buildx \ + CARGO_PROFILE=${{ inputs.cargo-profile }} \ + FEATURES=${{ inputs.features }} \ + BASE_IMAGE=${{ inputs.base-image }} + + - name: Upload artifacts + uses: ./.github/actions/upload-artifacts + with: + artifacts-dir: ${{ inputs.artifacts-dir }} + target-file: ./greptime + version: ${{ inputs.version }} + release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} diff --git a/.github/actions/build-greptime-images/action.yml b/.github/actions/build-greptime-images/action.yml new file mode 100644 index 0000000000..b4eaba302d --- /dev/null +++ b/.github/actions/build-greptime-images/action.yml @@ -0,0 +1,88 @@ +name: Build greptime images +description: Build and push greptime images +inputs: + image-registry: + description: The image registry to store the images + required: true + image-registry-username: + description: The username to login to the image registry + required: true + image-registry-password: + description: The password to login to the image registry + required: true + amd64-artifact-name: + description: The name of the amd64 artifact for building images + required: true + arm64-artifact-name: + description: The name of the arm64 artifact for building images + required: false + default: "" + image-namespace: + description: The namespace of the image registry to store the images + required: true + image-name: + description: The name of the image to build + required: true + image-tag: + description: The tag of the image to build + required: true + docker-file: + description: The path to the Dockerfile to build + required: true + platforms: + description: The supported platforms to build the image + required: true +runs: + using: composite + steps: + - name: Login to image registry + uses: docker/login-action@v2 + with: + registry: ${{ inputs.image-registry }} + username: ${{ inputs.image-registry-username }} + password: ${{ inputs.image-registry-password }} + + - name: Set up qemu for multi-platform builds + uses: docker/setup-qemu-action@v2 + + - name: Set up buildx + uses: docker/setup-buildx-action@v2 + + - name: Download amd64 artifacts + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.amd64-artifact-name }} + + - name: Unzip the amd64 artifacts + shell: bash + run: | + tar xvf ${{ inputs.amd64-artifact-name }}.tar.gz && \ + rm ${{ inputs.amd64-artifact-name }}.tar.gz && \ + rm -rf amd64 && \ + mv ${{ inputs.amd64-artifact-name }} amd64 + + - name: Download arm64 artifacts + uses: actions/download-artifact@v3 + if: ${{ inputs.arm64-artifact-name }} + with: + name: ${{ inputs.arm64-artifact-name }} + + - name: Unzip the arm64 artifacts + shell: bash + if: ${{ inputs.arm64-artifact-name }} + run: | + tar xvf ${{ inputs.arm64-artifact-name }}.tar.gz && \ + rm ${{ inputs.arm64-artifact-name }}.tar.gz && \ + rm -rf arm64 && \ + mv ${{ inputs.arm64-artifact-name }} arm64 + + - name: Build and push images for amd64 and arm64 + uses: docker/build-push-action@v3 + with: + context: . + file: ${{ inputs.docker-file }} + push: true + platforms: ${{ inputs.platforms }} + tags: | + ${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:latest + ${{ inputs.image-registry }}/${{ inputs.image-namespace }}/${{ inputs.image-name }}:${{ inputs.image-tag }} diff --git a/.github/actions/build-images/action.yml b/.github/actions/build-images/action.yml new file mode 100644 index 0000000000..4405e5c6b7 --- /dev/null +++ b/.github/actions/build-images/action.yml @@ -0,0 +1,47 @@ +name: Group for building greptimedb images +description: Group for building greptimedb images +inputs: + image-registry: + description: The image registry to store the images + required: true + image-namespace: + description: The namespace of the image registry to store the images + required: true + image-registry-username: + description: The username to login to the image registry + required: true + image-registry-password: + description: The password to login to the image registry + required: true + version: + description: Version of the artifact + required: true +runs: + using: composite + steps: + - name: Build and push standard images to dockerhub + uses: ./.github/actions/build-greptime-images + with: + image-registry: ${{ inputs.image-registry }} + image-namespace: ${{ inputs.image-namespace }} + image-registry-username: ${{ inputs.image-registry-username }} + image-registry-password: ${{ inputs.image-registry-password }} + image-name: greptimedb + image-tag: ${{ inputs.version }} + docker-file: docker/ci/Dockerfile + amd64-artifact-name: greptime-linux-amd64-pyo3-${{ inputs.version }} + arm64-artifact-name: greptime-linux-arm64-pyo3-${{ inputs.version }} + platforms: linux/amd64,linux/arm64 + + - name: Build and push centos images to dockerhub + uses: ./.github/actions/build-greptime-images + with: + image-registry: ${{ inputs.image-registry }} + image-namespace: ${{ inputs.image-namespace }} + image-registry-username: ${{ inputs.image-registry-username }} + image-registry-password: ${{ inputs.image-registry-password }} + image-name: greptimedb-centos + image-tag: ${{ inputs.version }} + docker-file: docker/ci/Dockerfile-centos + amd64-artifact-name: greptime-linux-amd64-centos-${{ inputs.version }} + platforms: linux/amd64 diff --git a/.github/actions/build-linux-artifacts/action.yml b/.github/actions/build-linux-artifacts/action.yml new file mode 100644 index 0000000000..1e82098455 --- /dev/null +++ b/.github/actions/build-linux-artifacts/action.yml @@ -0,0 +1,84 @@ +name: Build linux artifacts +description: Build linux artifacts +inputs: + arch: + description: Architecture to build + required: true + cargo-profile: + description: Cargo profile to build + required: true + version: + description: Version of the artifact + required: true + disable-run-tests: + description: Disable running integration tests + required: true + release-to-s3-bucket: + description: S3 bucket to store released artifacts + required: true + aws-access-key-id: + description: AWS access key id + required: true + aws-secret-access-key: + description: AWS secret access key + required: true + aws-region: + description: AWS region + required: true +runs: + using: composite + steps: + - name: Run integration test + if: ${{ inputs.disable-run-tests == 'false' }} + shell: bash + # NOTE: If the BUILD_JOBS > 4, it's always OOM in EC2 instance. + run: | + make run-it-in-container BUILD_JOBS=4 + + - name: Upload sqlness logs + if: ${{ failure() && inputs.disable-run-tests == 'false' }} # Only upload logs when the integration tests failed. + uses: actions/upload-artifact@v3 + with: + name: sqlness-logs + path: /tmp/greptime-*.log + retention-days: 3 + + - name: Build standard greptime + uses: ./.github/actions/build-greptime-binary + with: + base-image: ubuntu + features: pyo3_backend,servers/dashboard + cargo-profile: ${{ inputs.cargo-profile }} + artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }} + version: ${{ inputs.version }} + release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + - name: Build greptime without pyo3 + uses: ./.github/actions/build-greptime-binary + with: + base-image: ubuntu + features: servers/dashboard + cargo-profile: ${{ inputs.cargo-profile }} + artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }} + version: ${{ inputs.version }} + release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + - name: Build greptime on centos base image + uses: ./.github/actions/build-greptime-binary + if: ${{ inputs.arch == 'amd64' }} # Only build centos7 base image for amd64. + with: + base-image: centos + features: servers/dashboard + cargo-profile: ${{ inputs.cargo-profile }} + artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }} + version: ${{ inputs.version }} + release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} diff --git a/.github/actions/build-macos-artifacts/action.yml b/.github/actions/build-macos-artifacts/action.yml new file mode 100644 index 0000000000..024ec25a62 --- /dev/null +++ b/.github/actions/build-macos-artifacts/action.yml @@ -0,0 +1,105 @@ +name: Build macos artifacts +description: Build macos artifacts +inputs: + arch: + description: Architecture to build + required: true + rust-toolchain: + description: Rust toolchain to use + required: true + cargo-profile: + description: Cargo profile to build + required: true + features: + description: Cargo features to build + required: true + version: + description: Version of the artifact + required: true + disable-run-tests: + description: Disable running integration tests + required: true + release-to-s3-bucket: + description: S3 bucket to store released artifacts + required: true + artifacts-dir: + description: Directory to store artifacts + required: true + aws-access-key-id: + description: AWS access key id + required: true + aws-secret-access-key: + description: AWS secret access key + required: true + aws-region: + description: AWS region + required: true +runs: + using: composite + steps: + - name: Cache cargo assets + id: cache + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ inputs.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install protoc + shell: bash + run: | + brew install protobuf + + - name: Install rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ inputs.rust-toolchain }} + targets: ${{ inputs.arch }} + + - name: Start etcd # For integration tests. + if: ${{ inputs.disable-run-tests == 'false' }} + shell: bash + run: | + brew install etcd && \ + brew services start etcd + + - name: Install latest nextest release # For integration tests. + if: ${{ inputs.disable-run-tests == 'false' }} + uses: taiki-e/install-action@nextest + + - name: Run integration tests + if: ${{ inputs.disable_run_tests == 'false' }} + shell: bash + run: | + make test sqlness-test + + - name: Upload sqlness logs + if: ${{ failure() }} # Only upload logs when the integration tests failed. + uses: actions/upload-artifact@v3 + with: + name: sqlness-logs + path: /tmp/greptime-*.log + retention-days: 3 + + - name: Build greptime binary + shell: bash + run: | + make build \ + CARGO_PROFILE=${{ inputs.cargo-profile }} \ + FEATURES=${{ inputs.features }} \ + TARGET=${{ inputs.arch }} + + - name: Upload artifacts + uses: ./.github/actions/upload-artifacts + with: + artifacts-dir: ${{ inputs.artifacts-dir }} + target-file: target/${{ inputs.arch }}/${{ inputs.cargo-profile }}/greptime + version: ${{ inputs.version }} + release-to-s3-bucket: ${{ inputs.release-to-s3-bucket }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} diff --git a/.github/actions/release-artifacts/action.yml b/.github/actions/release-artifacts/action.yml new file mode 100644 index 0000000000..4b139ead55 --- /dev/null +++ b/.github/actions/release-artifacts/action.yml @@ -0,0 +1,50 @@ +name: Release artifacts +description: Release artifacts +inputs: + version: + description: Version to release + required: true +runs: + using: composite + steps: + # Download artifacts from previous jobs, the artifacts will be downloaded to: + # ${WORKING_DIR} + # |- greptime-darwin-amd64-pyo3-v0.5.0/greptime-darwin-amd64-pyo3-v0.5.0.tar.gz + # |- greptime-darwin-amd64-pyo3-v0.5.0.sha256sum/greptime-darwin-amd64-pyo3-v0.5.0.sha256sum + # |- greptime-darwin-amd64-v0.5.0/greptime-darwin-amd64-v0.5.0.tar.gz + # |- greptime-darwin-amd64-v0.5.0.sha256sum/greptime-darwin-amd64-v0.5.0.sha256sum + # ... + - name: Download artifacts + uses: actions/download-artifact@v3 + + - name: Create git tag for release + if: ${{ github.event_name != 'push' }} # Meaning this is a scheduled or manual workflow. + shell: bash + run: | + git tag ${{ inputs.version }} + + # Only publish release when the release tag is like v1.0.0, v1.0.1, v1.0.2, etc. + - name: Set release arguments + shell: bash + run: | + if [[ "${{ inputs.version }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "prerelease=false" >> $GITHUB_ENV + echo "makeLatest=true" >> $GITHUB_ENV + echo "generateReleaseNotes=false" >> $GITHUB_ENV + else + echo "prerelease=true" >> $GITHUB_ENV + echo "makeLatest=false" >> $GITHUB_ENV + echo "generateReleaseNotes=true" >> $GITHUB_ENV + fi + + - name: Publish release + uses: ncipollo/release-action@v1 + with: + name: "Release ${{ inputs.version }}" + prerelease: ${{ env.prerelease }} + makeLatest: ${{ env.makeLatest }} + tag: ${{ inputs.version }} + generateReleaseNotes: ${{ env.generateReleaseNotes }} + allowUpdates: true + artifacts: | + **/greptime-*/* diff --git a/.github/actions/start-runner/action.yml b/.github/actions/start-runner/action.yml new file mode 100644 index 0000000000..1b499e90f8 --- /dev/null +++ b/.github/actions/start-runner/action.yml @@ -0,0 +1,67 @@ +name: Start EC2 runner +description: Start EC2 runner +inputs: + runner: + description: The linux runner name + required: true + aws-access-key-id: + description: AWS access key id + required: true + aws-secret-access-key: + description: AWS secret access key + required: true + aws-region: + description: AWS region + required: true + github-token: + description: The GitHub token to clone private repository + required: false + default: "" + image-id: + description: The EC2 image id + required: true + security-group-id: + description: The EC2 security group id + required: true + subnet-id: + description: The EC2 subnet id + required: true +outputs: + label: + description: "label" + value: ${{ steps.start-linux-arm64-ec2-runner.outputs.label || inputs.runner }} + ec2-instance-id: + description: "ec2-instance-id" + value: ${{ steps.start-linux-arm64-ec2-runner.outputs.ec2-instance-id }} +runs: + using: composite + steps: + - name: Configure AWS credentials + if: startsWith(inputs.runner, 'ec2') + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + # The EC2 runner will use the following format: + # -- + # like 'ec2-c6a.4xlarge-amd64'. + - name: Get EC2 instance type + if: startsWith(inputs.runner, 'ec2') + id: get-ec2-instance-type + shell: bash + run: | + echo "instance-type=$(echo ${{ inputs.runner }} | cut -d'-' -f2)" >> $GITHUB_OUTPUT + + - name: Start EC2 runner + if: startsWith(inputs.runner, 'ec2') + uses: machulav/ec2-github-runner@v2 + id: start-linux-arm64-ec2-runner + with: + mode: start + ec2-image-id: ${{ inputs.image-id }} + ec2-instance-type: ${{ steps.get-ec2-instance-type.outputs.instance-type }} + subnet-id: ${{ inputs.subnet-id }} + security-group-id: ${{ inputs.security-group-id }} + github-token: ${{ inputs.github-token }} diff --git a/.github/actions/stop-runner/action.yml b/.github/actions/stop-runner/action.yml new file mode 100644 index 0000000000..f2a5f899d4 --- /dev/null +++ b/.github/actions/stop-runner/action.yml @@ -0,0 +1,41 @@ +name: Stop EC2 runner +description: Stop EC2 runner +inputs: + label: + description: The linux runner name + required: true + ec2-instance-id: + description: The EC2 instance id + required: true + aws-access-key-id: + description: AWS access key id + required: true + aws-secret-access-key: + description: AWS secret access key + required: true + aws-region: + description: AWS region + required: true + github-token: + description: The GitHub token to clone private repository + required: false + default: "" +runs: + using: composite + steps: + - name: Configure AWS credentials + if: ${{ inputs.label && inputs.ec2-instance-id }} + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + - name: Stop EC2 runner + if: ${{ inputs.label && inputs.ec2-instance-id }} + uses: machulav/ec2-github-runner@v2 + with: + mode: stop + label: ${{ inputs.label }} + ec2-instance-id: ${{ inputs.ec2-instance-id }} + github-token: ${{ inputs.github-token }} diff --git a/.github/actions/upload-artifacts/action.yml b/.github/actions/upload-artifacts/action.yml new file mode 100644 index 0000000000..4ab1a0acd1 --- /dev/null +++ b/.github/actions/upload-artifacts/action.yml @@ -0,0 +1,82 @@ +name: Upload artifacts +description: Upload artifacts +inputs: + artifacts-dir: + description: Directory to store artifacts + required: true + target-file: + description: The path of the target artifact + required: true + version: + description: Version of the artifact + required: true + release-to-s3-bucket: + description: S3 bucket to store released artifacts + required: true + aws-access-key-id: + description: AWS access key id + required: true + aws-secret-access-key: + description: AWS secret access key + required: true + aws-region: + description: AWS region + required: true +runs: + using: composite + steps: + - name: Create artifacts directory + shell: bash + run: | + mkdir -p ${{ inputs.artifacts-dir }} && \ + mv ${{ inputs.target-file }} ${{ inputs.artifacts-dir }} + + # The compressed artifacts will use the following layout: + # greptime-linux-amd64-pyo3-v0.3.0sha256sum + # greptime-linux-amd64-pyo3-v0.3.0.tar.gz + # greptime-linux-amd64-pyo3-v0.3.0 + # └── greptime + - name: Compress artifacts and calculate checksum + shell: bash + run: | + tar -zcvf ${{ inputs.artifacts-dir }}.tar.gz ${{ inputs.artifacts-dir }} && \ + echo $(shasum -a 256 ${{ inputs.artifacts-dir }}.tar.gz | cut -f1 -d' ') > ${{ inputs.artifacts-dir }}.sha256sum + + # Note: The artifacts will be double zip compressed(related issue: https://github.com/actions/upload-artifact/issues/39). + # However, when we use 'actions/download-artifact@v3' to download the artifacts, it will be automatically unzipped. + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.artifacts-dir }} + path: ${{ inputs.artifacts-dir }}.tar.gz + + - name: Upload checksum + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.artifacts-dir }}.sha256sum + path: ${{ inputs.artifacts-dir }}.sha256sum + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + - name: Upload artifacts to S3 + shell: bash + # The bucket layout will be: + # releases + # ├── v0.1.0 + # │ ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum + # │ └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz + # └── v0.2.0 + # ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum + # └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz + run: | + aws s3 cp \ + ${{ inputs.artifacts-dir }}.tar.gz \ + s3://${{ inputs.release-to-s3-bucket }}/releases/${{ inputs.version }}/${{ inputs.artifacts-dir }}.tar.gz && \ + aws s3 cp \ + ${{ inputs.artifacts-dir }}.sha256sum \ + s3://${{ inputs.release-to-s3-bucket }}/releases/${{ inputs.version }}/${{ inputs.artifacts-dir }}.sha256sum diff --git a/.github/scripts/create-version.sh b/.github/scripts/create-version.sh new file mode 100755 index 0000000000..22f0dd7105 --- /dev/null +++ b/.github/scripts/create-version.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -e + +# - If it's a tag push release, the version is the tag name(${{ github.ref_name }}); +# - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nigthly-20230313; +# - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-$(git rev-parse --short HEAD)-YYYYMMDDSS', like v0.2.0-e5b243c-2023071245; +# create_version ${GIHUB_EVENT_NAME} ${NEXT_RELEASE_VERSION} ${NIGHTLY_RELEASE_PREFIX} +function create_version() { + # Read from envrionment variables. + if [ -z "$GITHUB_EVENT_NAME" ]; then + echo "GITHUB_EVENT_NAME is empty" + exit 1 + fi + + if [ -z "$NEXT_RELEASE_VERSION" ]; then + echo "NEXT_RELEASE_VERSION is empty" + exit 1 + fi + + if [ -z "$NIGHTLY_RELEASE_PREFIX" ]; then + echo "NIGHTLY_RELEASE_PREFIX is empty" + exit 1 + fi + + # Note: Only output 'version=xxx' to stdout when everything is ok, so that it can be used in GitHub Actions Outputs. + if [ "$GITHUB_EVENT_NAME" = push ]; then + if [ -z "$GITHUB_REF_NAME" ]; then + echo "GITHUB_REF_NAME is empty in push event" + exit 1 + fi + echo "$GITHUB_REF_NAME" + elif [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + echo "$NEXT_RELEASE_VERSION-$(git rev-parse --short HEAD)-$(date "+%Y%m%d%S")" + elif [ "$GITHUB_EVENT_NAME" = schedule ]; then + echo "$NEXT_RELEASE_VERSION-$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")" + else + echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME" + exit 1 + fi +} + +# You can run as: GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigthly GITHUB_REF_NAME=v0.3.0 ./create-version.sh +create_version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 110b9a895d..a68c4a3107 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,3 +1,8 @@ +name: Release + +# There are two kinds of formal release: +# 1. The tag('v*.*.*') push release: the release workflow will be triggered by the tag push event. +# 2. The scheduled release(the version will be '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD'): the release workflow will be triggered by the schedule event. on: push: tags: @@ -5,522 +10,386 @@ on: schedule: # At 00:00 on Monday. - cron: '0 0 * * 1' - # Manually trigger only builds binaries. - workflow_dispatch: + workflow_dispatch: # Allows you to run this workflow manually. + # Notes: The GitHub Actions ONLY support 10 inputs, and it's already used up. inputs: - dry_run: - description: 'Skip docker push and release steps' + linux_amd64_runner: + type: choice + description: The runner uses to build linux-amd64 artifacts + default: ec2-c6i.4xlarge-amd64 + options: + - ubuntu-latest + - ubuntu-latest-8-cores + - ubuntu-latest-16-cores + - ubuntu-latest-32-cores + - ubuntu-latest-64-cores + - ec2-c6i.xlarge-amd64 # 4C8G + - ec2-c6i.2xlarge-amd64 # 8C16G + - ec2-c6i.4xlarge-amd64 # 16C32G + - ec2-c6i.8xlarge-amd64 # 32C64G + - ec2-c6i.16xlarge-amd64 # 64C128G + linux_arm64_runner: + type: choice + description: The runner uses to build linux-arm64 artifacts + default: ec2-c6g.4xlarge-arm64 + options: + - ec2-c6g.xlarge-arm64 # 4C8G + - ec2-c6g.2xlarge-arm64 # 8C16G + - ec2-c6g.4xlarge-arm64 # 16C32G + - ec2-c6g.8xlarge-arm64 # 32C64G + - ec2-c6g.16xlarge-arm64 # 64C128G + macos_runner: + type: choice + description: The runner uses to build macOS artifacts + default: macos-latest + options: + - macos-latest + skip_test: + description: Do not run integration tests during the build type: boolean default: true - skip_test: - description: 'Do not run tests during build' + build_linux_amd64_artifacts: type: boolean + description: Build linux-amd64 artifacts + required: false + default: false + build_linux_arm64_artifacts: + type: boolean + description: Build linux-arm64 artifacts + required: false + default: false + build_macos_artifacts: + type: boolean + description: Build macos artifacts + required: false + default: false + release_artifacts: + type: boolean + description: Create GitHub release and upload artifacts + required: false + default: false + release_images: + type: boolean + description: Build and push images to DockerHub and ACR + required: false + default: false + release_dev_builder_image: + type: boolean + description: Release dev-builder image + required: false default: false -name: Release - +# Use env variables to control all the release process. env: + # The arguments of building greptime. RUST_TOOLCHAIN: nightly-2023-05-03 - - SCHEDULED_BUILD_VERSION_PREFIX: v0.4.0 - - SCHEDULED_PERIOD: nightly - CARGO_PROFILE: nightly # Controls whether to run tests, include unit-test, integration-test and sqlness. - DISABLE_RUN_TESTS: ${{ inputs.skip_test || false }} + DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }} + + # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313; + NIGHTLY_RELEASE_PREFIX: nightly + # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release. + NEXT_RELEASE_VERSION: v0.4.0 jobs: - build-macos: - name: Build macOS binary + allocate-runners: + name: Allocate runners + if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} + runs-on: ubuntu-latest + outputs: + linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }} + linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }} + macos-runner: ${{ inputs.macos_runner || vars.DEFAULT_MACOS_RUNNER }} + + # The following EC2 resource id will be used for resource releasing. + linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }} + linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }} + linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }} + linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }} + + # The 'version' use as the global tag name of the release workflow. + version: ${{ steps.create-version.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # The create-version will create a global variable named 'version' in the global workflows. + # - If it's a tag push release, the version is the tag name(${{ github.ref_name }}); + # - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nigthly-20230313; + # - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}--YYYYMMDDSS', like v0.2.0-e5b243c-2023071245; + - name: Create version + id: create-version + run: | + echo "version=$(./.github/scripts/create-version.sh)" >> $GITHUB_OUTPUT + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_REF_NAME: ${{ github.ref_name }} + NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }} + NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }} + + - name: Allocate linux-amd64 runner + if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + uses: ./.github/actions/start-runner + id: start-linux-amd64-runner + with: + runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GITHUB_TOKEN }} + image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }} + security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }} + + - name: Allocate linux-arm64 runner + if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + uses: ./.github/actions/start-runner + id: start-linux-arm64-runner + with: + runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GITHUB_TOKEN }} + image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }} + security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }} + + build-linux-amd64-artifacts: + name: Build linux-amd64 artifacts + if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + ] + runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: ./.github/actions/build-linux-artifacts + with: + arch: amd64 + cargo-profile: ${{ env.CARGO_PROFILE }} + version: ${{ needs.allocate-runners.outputs.version }} + disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }} + aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }} + + build-linux-arm64-artifacts: + name: Build linux-arm64 artifacts + if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + ] + runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: ./.github/actions/build-linux-artifacts + with: + arch: arm64 + cargo-profile: ${{ env.CARGO_PROFILE }} + version: ${{ needs.allocate-runners.outputs.version }} + disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }} + aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }} + + build-macos-artifacts: + name: Build macOS artifacts strategy: fail-fast: false matrix: - # The file format is greptime-- include: - - arch: aarch64-apple-darwin - os: macos-latest - file: greptime-darwin-arm64 - continue-on-error: false - opts: "-F servers/dashboard" - - arch: x86_64-apple-darwin - os: macos-latest - file: greptime-darwin-amd64 - continue-on-error: false - opts: "-F servers/dashboard" - - arch: aarch64-apple-darwin - os: macos-latest - file: greptime-darwin-arm64-pyo3 - continue-on-error: false - opts: "-F pyo3_backend,servers/dashboard" - - arch: x86_64-apple-darwin - os: macos-latest - file: greptime-darwin-amd64-pyo3 - continue-on-error: false - opts: "-F pyo3_backend,servers/dashboard" + - os: ${{ needs.allocate-runners.outputs.macos-runner }} + arch: aarch64-apple-darwin + features: servers/dashboard + artifacts-dir-prefix: greptime-darwin-arm64 + - os: ${{ needs.allocate-runners.outputs.macos-runner }} + arch: aarch64-apple-darwin + features: pyo3_backend,servers/dashboard + artifacts-dir-prefix: greptime-darwin-arm64-pyo3 + - os: ${{ needs.allocate-runners.outputs.macos-runner }} + features: servers/dashboard + arch: x86_64-apple-darwin + artifacts-dir-prefix: greptime-darwin-amd64 + - os: ${{ needs.allocate-runners.outputs.macos-runner }} + features: pyo3_backend,servers/dashboard + arch: x86_64-apple-darwin + artifacts-dir-prefix: greptime-darwin-amd64-pyo3 runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.continue-on-error }} - if: github.repository == 'GreptimeTeam/greptimedb' + needs: [ + allocate-runners, + ] + if: ${{ inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Cache cargo assets - id: cache - uses: actions/cache@v3 + - uses: actions/checkout@v3 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }} + fetch-depth: 0 - - name: Install Protoc for macos - if: contains(matrix.arch, 'darwin') - run: | - brew install protobuf - - - name: Install etcd for macos - if: contains(matrix.arch, 'darwin') - run: | - brew install etcd - brew services start etcd - - - name: Install rust toolchain - uses: dtolnay/rust-toolchain@master + - uses: ./.github/actions/build-macos-artifacts with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - targets: ${{ matrix.arch }} - - name: Install latest nextest release - uses: taiki-e/install-action@nextest - - name: Output package versions - run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version + arch: ${{ matrix.arch }} + rust-toolchain: ${{ env.RUST_TOOLCHAIN }} + cargo-profile: ${{ env.CARGO_PROFILE }} + features: ${{ matrix.features }} + version: ${{ needs.allocate-runners.outputs.version }} + disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }} + artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }} + aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }} - - name: Run tests - if: env.DISABLE_RUN_TESTS == 'false' - run: make test sqlness-test - - - name: Run cargo build - if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false - run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }} - - - name: Calculate checksum and rename binary - shell: bash - run: | - cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }} - chmod +x greptime - tar -zcvf ${{ matrix.file }}.tgz greptime - echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.file }} - path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz - - - name: Upload checksum of artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.file }}.sha256sum - path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum - - - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD} - shell: bash - if: github.event_name != 'push' - run: | - buildTime=`date "+%Y%m%d"` - SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }} - echo "TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV - - - name: Configure tag - shell: bash - if: github.event_name == 'push' - run: | - VERSION=${{ github.ref_name }} - echo "TAG=${VERSION:1}" >> $GITHUB_ENV - - - name: Upload to S3 - run: | - aws s3 cp target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }} s3://${{ secrets.GREPTIMEDB_RELEASE_BUCKET_NAME }}/releases/${TAG} --recursive --exclude "*" --include "*.tgz" - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_CN_REGION }} - - build-linux: - name: Build linux binary - strategy: - fail-fast: false - matrix: - # The file format is greptime-- - include: - - arch: x86_64-unknown-linux-gnu - os: ubuntu-2004-16-cores - file: greptime-linux-amd64 - continue-on-error: false - opts: "-F servers/dashboard" - - arch: aarch64-unknown-linux-gnu - os: ubuntu-2004-16-cores - file: greptime-linux-arm64 - continue-on-error: false - opts: "-F servers/dashboard" - - arch: x86_64-unknown-linux-gnu - os: ubuntu-2004-16-cores - file: greptime-linux-amd64-pyo3 - continue-on-error: false - opts: "-F pyo3_backend,servers/dashboard" - - arch: aarch64-unknown-linux-gnu - os: ubuntu-2004-16-cores - file: greptime-linux-arm64-pyo3 - continue-on-error: false - opts: "-F pyo3_backend,servers/dashboard" - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.continue-on-error }} - if: github.repository == 'GreptimeTeam/greptimedb' + release-images-to-dockerhub: + name: Build and push images to DockerHub + if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + build-linux-amd64-artifacts, + build-linux-arm64-artifacts, + ] + runs-on: ubuntu-2004-16-cores steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Cache cargo assets - id: cache - uses: actions/cache@v3 + - uses: actions/checkout@v3 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }} + fetch-depth: 0 - - name: Install Protoc for linux - if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu') - run: | # Make sure the protoc is >= 3.15 - wget https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip - unzip protoc-21.9-linux-x86_64.zip -d protoc - sudo cp protoc/bin/protoc /usr/local/bin/ - sudo cp -r protoc/include/google /usr/local/include/ - - - name: Install etcd for linux - if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu') - run: | - ETCD_VER=v3.5.7 - DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download - curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz - mkdir -p /tmp/etcd-download - tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1 - rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz - - sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/ - nohup etcd >/tmp/etcd.log 2>&1 & - - - name: Install dependencies for linux - if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu') - run: | - sudo apt-get -y update - sudo apt-get -y install libssl-dev pkg-config g++-aarch64-linux-gnu gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu wget - - # FIXME(zyy17): Should we specify the version of python when building binary for darwin? - - name: Compile Python 3.10.10 from source for linux - if: contains(matrix.arch, 'linux') && contains(matrix.opts, 'pyo3_backend') - run: | - sudo chmod +x ./docker/aarch64/compile-python.sh - sudo ./docker/aarch64/compile-python.sh ${{ matrix.arch }} - - - name: Install rust toolchain - uses: dtolnay/rust-toolchain@master + - name: Build and push images to dockerhub + uses: ./.github/actions/build-images with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - targets: ${{ matrix.arch }} - - name: Install latest nextest release - uses: taiki-e/install-action@nextest - - name: Output package versions - run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version + image-registry: docker.io + image-namespace: ${{ vars.IMAGE_NAMESPACE }} + image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }} + image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }} + version: ${{ needs.allocate-runners.outputs.version }} - - name: Run tests - if: env.DISABLE_RUN_TESTS == 'false' - run: make test sqlness-test - - - name: Run cargo build - if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false - run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }} - - - name: Run cargo build with pyo3 for aarch64-linux - if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend') - run: | - # TODO(zyy17): We should make PYO3_CROSS_LIB_DIR configurable. - export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64 - export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH - export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH - export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH - - export PYO3_CROSS_LIB_DIR=${PWD}/python-3.10.10/aarch64 - echo "PYO3_CROSS_LIB_DIR: $PYO3_CROSS_LIB_DIR" - alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3 - alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip - - cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }} - - - name: Run cargo build with pyo3 for amd64-linux - if: contains(matrix.arch, 'x86_64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend') - run: | - export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64 - export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH - export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH - export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH - - echo "implementation=CPython" >> pyo3.config - echo "version=3.10" >> pyo3.config - echo "implementation=CPython" >> pyo3.config - echo "shared=true" >> pyo3.config - echo "abi3=true" >> pyo3.config - echo "lib_name=python3.10" >> pyo3.config - echo "lib_dir=$PYTHON_INSTALL_PATH_AMD64/lib" >> pyo3.config - echo "executable=$PYTHON_INSTALL_PATH_AMD64/bin/python3" >> pyo3.config - echo "pointer_width=64" >> pyo3.config - echo "build_flags=" >> pyo3.config - echo "suppress_build_script_link_lines=false" >> pyo3.config - - cat pyo3.config - export PYO3_CONFIG_FILE=${PWD}/pyo3.config - alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3 - alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip - - cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }} - - - name: Calculate checksum and rename binary - shell: bash - run: | - cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }} - chmod +x greptime - tar -zcvf ${{ matrix.file }}.tgz greptime - echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.file }} - path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz - - - name: Upload checksum of artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.file }}.sha256sum - path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum - - - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD} - shell: bash - if: github.event_name != 'push' - run: | - buildTime=`date "+%Y%m%d"` - SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }} - echo "TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV - - - name: Configure tag - shell: bash - if: github.event_name == 'push' - run: | - VERSION=${{ github.ref_name }} - echo "TAG=${VERSION:1}" >> $GITHUB_ENV - - - name: Upload to S3 - run: | - aws s3 cp target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }} s3://${{ secrets.GREPTIMEDB_RELEASE_BUCKET_NAME }}/releases/${TAG} --recursive --exclude "*" --include "*.tgz" - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_CN_REGION }} - - docker: - name: Build docker image - needs: [build-linux, build-macos] - runs-on: ubuntu-latest - if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false) - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD} - shell: bash - if: github.event_name != 'push' - run: | - buildTime=`date "+%Y%m%d"` - SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }} - echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV - - - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0. - shell: bash - if: github.event_name == 'push' - run: | - VERSION=${{ github.ref_name }} - echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up buildx - uses: docker/setup-buildx-action@v2 - - - name: Download amd64 binary - uses: actions/download-artifact@v3 - with: - name: greptime-linux-amd64-pyo3 - path: amd64 - - - name: Unzip the amd64 artifacts - run: | - tar xvf amd64/greptime-linux-amd64-pyo3.tgz -C amd64/ && rm amd64/greptime-linux-amd64-pyo3.tgz - cp -r amd64 docker/ci - - - name: Download arm64 binary - id: download-arm64 - uses: actions/download-artifact@v3 - with: - name: greptime-linux-arm64-pyo3 - path: arm64 - - - name: Unzip the arm64 artifacts - id: unzip-arm64 - if: success() || steps.download-arm64.conclusion == 'success' - run: | - tar xvf arm64/greptime-linux-arm64-pyo3.tgz -C arm64/ && rm arm64/greptime-linux-arm64-pyo3.tgz - cp -r arm64 docker/ci - - - name: Build and push all - uses: docker/build-push-action@v3 - if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds - with: - context: . - file: ./docker/ci/Dockerfile - push: true - platforms: linux/amd64,linux/arm64 - tags: | - greptime/greptimedb:latest - greptime/greptimedb:${{ env.IMAGE_TAG }} - - - name: Build and push amd64 only - uses: docker/build-push-action@v3 - if: success() || steps.download-arm64.conclusion == 'failure' # Only build and push amd64 platform if download-arm64 fails - with: - context: . - file: ./docker/ci/Dockerfile - push: true - platforms: linux/amd64 - tags: | - greptime/greptimedb:latest - greptime/greptimedb:${{ env.IMAGE_TAG }} - - release: - name: Release artifacts - # Release artifacts only when all the artifacts are built successfully. - needs: [build-linux, build-macos, docker] - runs-on: ubuntu-latest - if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false) - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Download artifacts - uses: actions/download-artifact@v3 - - - name: Configure scheduled build version # the version would be ${SCHEDULED_BUILD_VERSION_PREFIX}-${SCHEDULED_PERIOD}-YYYYMMDD, like v0.2.0-nigthly-20230313. - shell: bash - if: github.event_name != 'push' - run: | - buildTime=`date "+%Y%m%d"` - SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-${{ env.SCHEDULED_PERIOD }}-$buildTime - echo "SCHEDULED_BUILD_VERSION=${SCHEDULED_BUILD_VERSION}" >> $GITHUB_ENV - - # Only publish release when the release tag is like v1.0.0, v1.0.1, v1.0.2, etc. - - name: Set whether it is the latest release - run: | - if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "prerelease=false" >> $GITHUB_ENV - echo "makeLatest=true" >> $GITHUB_ENV - else - echo "prerelease=true" >> $GITHUB_ENV - echo "makeLatest=false" >> $GITHUB_ENV - fi - - - name: Create scheduled build git tag - if: github.event_name != 'push' - run: | - git tag ${{ env.SCHEDULED_BUILD_VERSION }} - - - name: Publish scheduled release # configure the different release title and tags. - uses: ncipollo/release-action@v1 - if: github.event_name != 'push' - with: - name: "Release ${{ env.SCHEDULED_BUILD_VERSION }}" - prerelease: ${{ env.prerelease }} - makeLatest: ${{ env.makeLatest }} - tag: ${{ env.SCHEDULED_BUILD_VERSION }} - generateReleaseNotes: true - artifacts: | - **/greptime-* - - - name: Publish release - uses: ncipollo/release-action@v1 - if: github.event_name == 'push' - with: - name: "${{ github.ref_name }}" - prerelease: ${{ env.prerelease }} - makeLatest: ${{ env.makeLatest }} - generateReleaseNotes: false - allowUpdates: true - artifacts: | - **/greptime-* - - docker-push-acr: - name: Push docker image to alibaba cloud container registry - needs: [docker] - runs-on: ubuntu-latest - if: github.repository == 'GreptimeTeam/greptimedb' && !(inputs.dry_run || false) + release-images-to-acr: + name: Build and push images to ACR + if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + build-linux-amd64-artifacts, + build-linux-arm64-artifacts, + ] + runs-on: ubuntu-2004-16-cores + # When we push to ACR, it's easy to fail due to some unknown network issues. + # However, we don't want to fail the whole workflow because of this. + # The ACR have daily sync with DockerHub, so don't worry about the image not being updated. continue-on-error: true steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to alibaba cloud container registry - uses: docker/login-action@v2 + - uses: actions/checkout@v3 with: - registry: greptime-registry.cn-hangzhou.cr.aliyuncs.com - username: ${{ secrets.ALICLOUD_USERNAME }} - password: ${{ secrets.ALICLOUD_PASSWORD }} + fetch-depth: 0 - - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD} - shell: bash - if: github.event_name != 'push' - run: | - buildTime=`date "+%Y%m%d"` - SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }} - echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV + - name: Build and push images to ACR + uses: ./.github/actions/build-images + with: + image-registry: ${{ vars.ACR_IMAGE_REGISTRY }} + image-namespace: ${{ vars.IMAGE_NAMESPACE }} + image-registry-username: ${{ secrets.ALICLOUD_USERNAME }} + image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }} + version: ${{ needs.allocate-runners.outputs.version }} - - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0. - shell: bash - if: github.event_name == 'push' - run: | - VERSION=${{ github.ref_name }} - echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV + release-artifacts: + name: Create GitHub release and upload artifacts + if: ${{ inputs.release_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + build-linux-amd64-artifacts, + build-linux-arm64-artifacts, + build-macos-artifacts, + release-images-to-dockerhub, + ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 - - name: Push image to alibaba cloud container registry # Use 'docker buildx imagetools create' to create a new image base on source image. - run: | - docker buildx imagetools create \ - --tag greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb:latest \ - --tag greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb:${{ env.IMAGE_TAG }} \ - greptime/greptimedb:${{ env.IMAGE_TAG }} + - name: Release artifacts + uses: ./.github/actions/release-artifacts + with: + version: ${{ needs.allocate-runners.outputs.version }} + + release-dev-builder-image: + name: Release dev builder image + if: ${{ inputs.release_dev_builder_image }} # Only manually trigger this job. + runs-on: ubuntu-latest-16-cores + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build and push dev builder image + uses: ./.github/actions/build-dev-builder-image + with: + dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub-image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }} + acr-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }} + acr-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }} + acr-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }} + + ### Stop runners ### + # It's very necessary to split the job of releasing runners into 'stop-linux-amd64-runner' and 'stop-linux-arm64-runner'. + # Because we can terminate the specified EC2 instance immediately after the job is finished without uncessary waiting. + stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released. + name: Stop linux-amd64 runner + # Only run this job when the runner is allocated. + if: ${{ always() }} + runs-on: ubuntu-latest + needs: [ + allocate-runners, + build-linux-amd64-artifacts, + ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Stop EC2 runner + uses: ./.github/actions/stop-runner + with: + label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }} + ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released. + name: Stop linux-arm64 runner + # Only run this job when the runner is allocated. + if: ${{ always() }} + runs-on: ubuntu-latest + needs: [ + allocate-runners, + build-linux-arm64-artifacts, + ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Stop EC2 runner + uses: ./.github/actions/stop-runner + with: + label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }} + ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/docker/centos/Dockerfile b/docker/centos/Dockerfile index ae42114593..843fd827f8 100644 --- a/docker/centos/Dockerfile +++ b/docker/centos/Dockerfile @@ -26,7 +26,7 @@ ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH # Build the project in release mode. RUN --mount=target=.,rw \ - --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/root/.cargo/registry \ make build \ CARGO_PROFILE=${CARGO_PROFILE} \ FEATURES=${FEATURES} \ diff --git a/docker/ubuntu/Dockerfile b/docker/ubuntu/Dockerfile index 18cdd7e168..cdb09f774e 100644 --- a/docker/ubuntu/Dockerfile +++ b/docker/ubuntu/Dockerfile @@ -7,7 +7,8 @@ ENV LANG en_US.utf8 WORKDIR /greptimedb # Install dependencies. -RUN apt-get update && apt-get install -y \ +RUN --mount=type=cache,target=/var/cache/apt \ + apt-get update && apt-get install -y \ libssl-dev \ protobuf-compiler \ curl \ @@ -25,7 +26,7 @@ ENV PATH /root/.cargo/bin/:$PATH # Build the project in release mode. RUN --mount=target=.,rw \ - --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/root/.cargo/registry \ make build \ CARGO_PROFILE=${CARGO_PROFILE} \ FEATURES=${FEATURES} \