Only run the necessary actions on /docs changes (#10753)

- Move pull request checks for components to a separate workflow that
  runs on /components changes
- Limit publish-components workflow runs to only on /components changes
- Limit publish-marketplace-content workflow runs to only on /components
  changes
- Change the publish-packages workflow to also run on pull-requests,
  excluding /docs changes. When running on pull-requests, do a dry run
  of publishing packages.

.github/workflows/components-pr.yaml

@@ -0,0 +1,200 @@
+# TODO combine this with publish-components.yml
+name: Components Checks
+
+on:
+ pull_request:
+ branches:
+ - master
+ paths:
+ - 'components/**'
+
+jobs:
+ check_version:
+ name: Ensure component commits modify component versions
+ runs-on: ubuntu-latest
+
+ permissions: 
+ contents: write
+ pull-requests: write
+
+ steps:
+ - uses: actions/[email protected]
+ name: Checkout repo
+ with:
+ # See https://github.com/actions/checkout#checkout-v2
+ # This will be slow. The intent is to fetch all commits
+ # since the merge-base (the commit where we branched off)
+ # so we can check the git diff against all changed files.
+ # By default, the checkout action only returns the last commit,
+ # There's no native way to do this in the checkout action, so
+ # we have to fetch the entire history. See
+ # https://github.com/actions/checkout/issues/266#issuecomment-638346893
+ fetch-depth: 0
+ - uses: jitterbit/get-changed-files@v1
+ id: changed_files
+ name: Get changed files
+ with:
+ format: json
+ - name: Check git diff for version changes
+ uses: ./.github/actions/git-diff-on-components
+ with:
+ all_files: ${{ steps.changed_files.outputs.all }}
+ base_commit: ${{ github.event.pull_request.base.sha }}
+ head_commit: ${{ github.event.pull_request.head.sha }}
+
+ verify-typescript-components:
+ name: Verify TypeScript components
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/[email protected]
+ - uses: pnpm/[email protected]
+ with:
+ version: 7.33.6
+ - name: Get pnpm store directory
+ id: pnpm-cache
+ run: |
+ echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+ - uses: actions/cache@v4
+ name: Setup pnpm cache
+ with:
+ path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+ key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+ restore-keys: |
+ ${{ runner.os }}-pnpm-store-
+ - name: Setup Node Env
+ uses: actions/[email protected]
+ with:
+ node-version: 18
+ registry-url: https://registry.npmjs.org/
+ cache: 'pnpm'
+ - name: Install Dependencies
+ run: pnpm install -r
+ - name: Compile TypeScript
+ id: compile
+ run: npm run build > files.txt
+ - name: Get Changed Files
+ id: files
+ uses: jitterbit/get-changed-files@v1
+ with:
+ format: 'csv'
+ - name: Check For Compiled TypeScript Files
+ run: |
+ IFS=$'\n'
+ # Remove initial tsc output
+ output_files=$(cat files.txt | sed 1,3d)
+ declare -a ERRORS
+ declare -a SKIPPED
+ mapfile -d ',' -t added_modified_renamed_files < <(printf '%s,%s' '${{ steps.files.outputs.added_modified }}' '${{ steps.files.outputs.renamed }}')
+ for added_modified_file in "${added_modified_renamed_files[@]}";
+ do
+ echo "Checking if $added_modified_file is a publishable ts file"
+ if [[ $added_modified_file == components/* ]] && [[ $added_modified_file == *.ts ]] && [[ $added_modified_file != *.app.ts ]] \
+ && [[ $added_modified_file != */common*.ts ]] && [[ $added_modified_file != */common/* ]]
+ then
+ # XXX This is a hacky way to publish only TS components with changes. If a changed
+ # file "path-a/path-b/c.ts" has a corresponding compiled output file
+ # "path-a/dist/path-b/c.mjs", attempt to publish the output `.mjs` file.
+ changed_output_file=""
+ for f in $output_files; # check each output file for a match
+ do
+ # Replaces /dist/path/filename.mjs with /path/filename.ts
+ maybe_source_file=$(echo "$f" | sed 's/\/dist\//\//;s/.mjs/\.ts/')
+ if [[ ${maybe_source_file} == **/${added_modified_file} ]]
+ then
+ changed_output_file=${f}
+ break
+ fi
+ done
+ if [[ $changed_output_file == "" ]]
+ then
+ ERROR_MESSAGE="cannot find an output .mjs file with ${added_modified_file}"
+ echo $ERROR_MESSAGE
+ ERRORS+=("*${ERROR_MESSAGE}")
+ fi
+ else
+ echo "$added_modified_file will not be added to the registry"
+ SKIPPED+=("*$added_modified_file")
+ fi
+ done
+ if [[ ${#SKIPPED[@]} -ne 0 ]]; then
+ echo "the following files were skipped:"
+ printf '%s\n' "${SKIPPED[@]}"
+ fi
+ if [[ ${#ERRORS[@]} -ne 0 ]]; then
+ echo "the following files generated errors:"
+ printf '%s\n' "${ERRORS[@]}"
+ echo "Please check if the components above were successfully compiled"
+ echo "More information here: https://pipedream.com/docs/components/typescript/#developing-typescript-components-in-the-pipedreamhq-pipedream-registry"
+ exit 1
+ fi
+ unset IFS
+
+ publish-typescript-components-dry-run:
+ name: Publish TypeScript components
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout code
+ uses: actions/[email protected]
+ - uses: pnpm/[email protected]
+ with:
+ version: 7.33.6
+ - name: Get pnpm store directory
+ id: pnpm-cache
+ run: |
+ echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+ - uses: actions/cache@v4
+ name: Setup pnpm cache
+ with:
+ path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+ key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+ restore-keys: |
+ ${{ runner.os }}-pnpm-store-
+ - name: Setup Node Env
+ uses: actions/[email protected]
+ with:
+ node-version: 18
+ registry-url: https://registry.npmjs.org/
+ cache: 'pnpm'
+ - name: Install Dependencies
+ run: pnpm install -r
+ - name: Compile TypeScript
+ id: compile
+ run: npm run build > files.txt
+ - name: Get Changed Files
+ id: files
+ uses: jitterbit/get-changed-files@v1
+ with:
+ format: 'csv'
+ - name: Publish TypeScript components (dry run)
+ shell: bash {0} # don't fast fail
+ run: |
+ IFS=$'\n'
+ mapfile -d ',' -t added_modified_renamed_files < <(printf '%s,%s' '${{ steps.files.outputs.added_modified }}' '${{ steps.files.outputs.renamed }}')
+ # Remove initial tsc output
+ output_files=$(cat files.txt | sed 1,3d)
+ echo "The following files will be published on merge:"
+ for added_modified_file in "${added_modified_renamed_files[@]}";
+ do
+ # starts with components, ends with .ts and not app.ts, doesn't end with /common*.ts,
+ # and doesn't follow */common/
+ if [[ $added_modified_file == components/* ]] && [[ $added_modified_file == *.ts ]] && [[ $added_modified_file != *.app.ts ]] \
+ && [[ $added_modified_file != */common*.ts ]] && [[ $added_modified_file != */common/* ]]
+ then
+ # XXX This is a hacky way to publish only TS components with changes. If a changed
+ # file "path-a/path-b/c.ts" has a corresponding compiled output file
+ # "path-a/dist/path-b/c.mjs", attempt to publish the output `.mjs` file.
+ for f in $output_files;
+ do
+ # Replaces /dist/path/filename.mjs with /path/filename.ts
+ maybe_source_file=$(echo "$f" | sed 's/\/dist\//\//;s/.mjs/\.ts/')
+ if [[ ${maybe_source_file} == **/${added_modified_file} ]]
+ then
+ echo "$f"
+ fi
+ done
+ fi
+ done
+ unset IFS

.github/workflows/publish-components.yaml

@@ -2,6 +2,8 @@ on:
  push:
  branches:
  - master
+ paths:
+ - 'components/**'
 
 jobs:
  publish-components:

.github/workflows/publish-marketplace-content.yaml

@@ -2,6 +2,8 @@ on:
  push:
  branches:
  - master
+ paths:
+ - 'components/**'
 
 jobs:
  publish-components:

.github/workflows/publish-packages.yaml

@@ -2,19 +2,42 @@ on:
  push:
  branches:
  - master
+ paths-ignore:
+ - 'docs/**'
+ - 'docs-v2/**'
+ pull_request:
+ branches:
+ - master
+ paths-ignore:
+ - 'docs/**'
+ - 'docs-v2/**'
 
 jobs:
  # See https://pnpm.io/continuous-integration#github-actions
  publish:
+ name: pnpm publish
  runs-on: ubuntu-latest
  steps:
  - uses: actions/[email protected]
  - uses: pnpm/[email protected]
  with:
  version: 7.33.6
+ - name: Get pnpm store directory
+ if: github.ref != 'refs/heads/master' # Cache is used only for dry runs
+ id: pnpm-cache
+ run: |
+ echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+ - uses: actions/cache@v4
+ if: github.ref != 'refs/heads/master' # Cache is used only for dry runs
+ name: Setup pnpm cache
+ with:
+ path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+ key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+ restore-keys: |
+ ${{ runner.os }}-pnpm-store-
  - uses: actions/[email protected]
  with:
- node-version: 14
+ node-version: 18
  registry-url: https://registry.npmjs.org/
  cache: 'pnpm'
  - name: pnpm install
@@ -24,5 +47,9 @@ jobs:
  # See https://pnpm.io/using-changesets
  - name: Setup npmrc for pnpm publish
  run: echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > .npmrc
+ - name: pnpm publish (dry run)
+ if: github.ref != 'refs/heads/master'
+ run: pnpm publish -r --no-git-checks --dry-run
  - name: pnpm publish
+ if: github.ref == 'refs/heads/master'
  run: pnpm publish -r --no-git-checks