diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh
index 03fad62d331..ebe4a31dd89 100755
--- a/src/ci/scripts/should-skip-this.sh
+++ b/src/ci/scripts/should-skip-this.sh
@@ -6,15 +6,19 @@
 set -euo pipefail
 IFS=$'\n\t'
 
+git fetch "https://github.com/$GITHUB_REPOSITORY" "$GITHUB_BASE_REF"
+BASE_COMMIT="$(git merge-base FETCH_HEAD HEAD)"
+
+echo "Searching for toolstate changes between $BASE_COMMIT and $(git rev-parse HEAD)"
 source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
 
 if [[ -z "${CI_ONLY_WHEN_SUBMODULES_CHANGED+x}" ]]; then
     echo "Executing the job since there is no skip rule in effect"
-elif git diff HEAD^ | grep --quiet "^index .* 160000"; then
+elif git diff "$BASE_COMMIT" | grep --quiet "^index .* 160000"; then
     # Submodules pseudo-files inside git have the 160000 permissions, so when
     # those files are present in the diff a submodule was updated.
     echo "Executing the job since submodules are updated"
-elif git diff --name-only HEAD^ | grep --quiet src/tools/'\(clippy\|rustfmt\)'; then
+elif git diff --name-only "$BASE_COMMIT" | grep --quiet src/tools/'\(clippy\|rustfmt\)'; then
     # There is not an easy blanket search for subtrees. For now, manually list
     # clippy.
     echo "Executing the job since clippy or rustfmt subtree was updated"