diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 29d712c681ef..d77009aa5eb8 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -33,26 +33,17 @@ if [ -f "${KOKORO_GFILE_DIR}/secret_manager/java-bigqueryconnection-samples-secr source "${KOKORO_GFILE_DIR}/secret_manager/java-bigqueryconnection-samples-secrets" fi -if [[ -n "${BUILD_SUBDIR}" ]] -then - echo "Compiling and building all modules for ${BUILD_SUBDIR}" - mvn clean install \ - -DskipTests \ - -Dclirr.skip \ - -Dflatten.skip \ - -Dcheckstyle.skip \ - -Djacoco.skip \ - -Denforcer.skip \ - --also-make \ - --projects "${BUILD_SUBDIR}" - echo "Running in subdir: ${BUILD_SUBDIR}" - pushd "${BUILD_SUBDIR}" -fi - RETURN_CODE=0 case ${JOB_TYPE} in test) + if [[ -n "${BUILD_SUBDIR}" ]] + then + echo "Compiling and building all modules for ${BUILD_SUBDIR}" + install_modules "${BUILD_SUBDIR}" + echo "Running in subdir: ${BUILD_SUBDIR}" + pushd "${BUILD_SUBDIR}" + fi echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}" retry_with_backoff 3 10 \ mvn test \ @@ -67,6 +58,12 @@ case ${JOB_TYPE} in -Dmaven.wagon.http.retryHandler.count=5 \ -T 1C ${SUREFIRE_JVM_OPT} RETURN_CODE=$? + + if [[ -n "${BUILD_SUBDIR}" ]] + then + echo "restoring directory" + popd + fi echo "Finished running unit tests" ;; integration) @@ -85,6 +82,41 @@ case ${JOB_TYPE} in echo "No Integration Tests to run" fi ;; + integration-single) + generate_modified_modules_list false + if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then + echo "Not running integration checks -- this is Release Please SNAPSHOT pull request." + elif [[ ! " ${modified_module_list[*]} " =~ " ${BUILD_SUBDIR} " ]]; then + echo "${BUILD_SUBDIR} not modified, skipping split integration test" + else + echo "${BUILD_SUBDIR} modified, running split integration test" + echo "Compiling and building all modules for ${BUILD_SUBDIR}" + install_modules "${BUILD_SUBDIR}" + echo "Running in subdir: ${BUILD_SUBDIR}" + pushd "${BUILD_SUBDIR}" + echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}" + echo "INTEGRATION_TEST_ARGS: ${INTEGRATION_TEST_ARGS}" + mvn verify -Penable-integration-tests \ + ${INTEGRATION_TEST_ARGS} \ + -B -ntp -fae \ + -DtrimStackTrace=false \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dorg.slf4j.simpleLogger.showDateTime=true \ + -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ + -Dcheckstyle.skip=true \ + -Dflatten.skip=true \ + -Danimal.sniffer.skip=true \ + -Djacoco.skip=true \ + -DskipUnitTests=true \ + -Dmaven.wagon.http.retryHandler.count=5 \ + -T 1C ${SUREFIRE_JVM_OPT} + + RETURN_CODE=$? + popd + printf "Finished integration tests for modules:\n%s\n" "${BUILD_SUBDIR}" + fi + ;; graalvm-presubmit) generate_graalvm_presubmit_modules_list if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then @@ -109,7 +141,44 @@ case ${JOB_TYPE} in echo "Not running GraalVM checks -- No changes in relevant modules" fi ;; + graalvm-single) + generate_modified_modules_list false + if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then + echo "Not running GraalVM checks -- this is Release Please SNAPSHOT pull request." + elif [[ ! " ${modified_module_list[*]} " =~ " ${BUILD_SUBDIR} " ]]; then + echo "${BUILD_SUBDIR} not modified, skipping split GraalVM test" + else + echo "${BUILD_SUBDIR} modified, running split GraalVM test" + echo "Compiling and building all modules for ${BUILD_SUBDIR}" + install_modules "${BUILD_SUBDIR}" + echo "Running in subdir: ${BUILD_SUBDIR}" + pushd "${BUILD_SUBDIR}" + echo "INTEGRATION_TEST_ARGS: ${INTEGRATION_TEST_ARGS}" + mvn test -Pnative \ + ${INTEGRATION_TEST_ARGS} \ + -B -ntp -fae \ + -DtrimStackTrace=false \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dorg.slf4j.simpleLogger.showDateTime=true \ + -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ + -Dcheckstyle.skip=true \ + -Dflatten.skip=true \ + -Danimal.sniffer.skip=true + + RETURN_CODE=$? + popd + printf "Finished GraalVM ITs for modules:\n%s\n" "${BUILD_SUBDIR}" + fi + ;; lint) + if [[ -n "${BUILD_SUBDIR}" ]] + then + echo "Compiling and building all modules for ${BUILD_SUBDIR}" + install_modules "${BUILD_SUBDIR}" + echo "Running in subdir: ${BUILD_SUBDIR}" + pushd "${BUILD_SUBDIR}" + fi if [ -n "${BASE_SHA}" ] && [ -n "${HEAD_SHA}" ]; then changed_file_list=$(git diff --name-only "${BASE_SHA}" "${HEAD_SHA}") echo "${changed_file_list}" @@ -137,17 +206,17 @@ case ${JOB_TYPE} in -T 1.5C \ com.spotify.fmt:fmt-maven-plugin:check mvn -B -ntp checkstyle:check@checkstyle + + if [[ -n "${BUILD_SUBDIR}" ]] + then + echo "restoring directory" + popd + fi ;; *) ;; esac -if [[ -n "${BUILD_SUBDIR}" ]] -then - echo "restoring directory" - popd -fi - if [ "${REPORT_COVERAGE}" == "true" ]; then bash ${KOKORO_GFILE_DIR}/codecov.sh fi diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 4f8959c245ef..5531e83667e3 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -13,7 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-vertexai') +excluded_modules=( + 'gapic-libraries-bom' + 'google-cloud-jar-parent' + 'google-cloud-pom-parent' + 'java-vertexai', + 'java-logging' +) function retry_with_backoff { attempts_left=$1 @@ -56,10 +62,8 @@ function retry_with_backoff { # Given a folder containing a maven multi-module, assign the variable 'submodules' to a # comma-delimited list of /. function parse_submodules() { - pushd "$1" >/dev/null - submodules_array=() - mvn_submodules=$(mvn help:evaluate -Dexpression=project.modules) + mvn_submodules=$(mvn help:evaluate -Dexpression=project.modules -pl "$1") if mvn_submodules=$(grep '<.*>.*' <<< "$mvn_submodules"); then mvn_submodules=$(sed -e 's/<.*>\(.*\)<\/.*>/\1/g' <<< "$mvn_submodules") for submodule in $mvn_submodules; do @@ -77,8 +81,6 @@ function parse_submodules() { echo "${submodules_array[*]}" ) export submodules - - popd >/dev/null } # Given a list of folders containing maven multi-modules, assign the variable 'all_submodules' to a @@ -155,6 +157,13 @@ function release_please_snapshot_pull_request() { fi } +# Sets bash variables for maven_modules and modified_module_list +# maven_modules is the list of all maven submodules of the root pom +# modified_module_list is the subset of maven_modules that have been touched +# in the current pull request +# +# The first positional argument is a value true/false. If true (default), then +# exclude modules from the global exclusion list. function generate_modified_modules_list() { # Find the files changed from when the PR branched to the last commit # Filter for java modules and get all the unique elements @@ -167,11 +176,19 @@ function generate_modified_modules_list() { # Generate the list of valid maven modules maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g') maven_modules=() - for module in $maven_modules_list; do - if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then - maven_modules+=("${module}") - fi - done + + # If the first argument is "true" (default), then use the module exclusion list + use_exclusion_list=${1:-true} + if [[ "${use_exclusion_list}" == "true" ]]; then + echo "Excluding modules from the global exclusion list" + for module in $maven_modules_list; do + if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then + maven_modules+=("${module}") + fi + done + else + maven_modules=(${maven_modules_list[*]}) + fi modified_module_list=() # If either parent pom.xml is touched, run ITs on all the modules diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index 1ea066f5bf77..5805a07b9a18 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -51,6 +51,8 @@ export MAVEN_OPTS=$(determineMavenOpts) if [[ -n "${BUILD_SUBDIR}" ]] then + echo "Compiling and building all modules for ${BUILD_SUBDIR}" + install_dependencies "${BUILD_SUBDIR}" echo "Running in subdir: ${BUILD_SUBDIR}" pushd "${BUILD_SUBDIR}" fi diff --git a/.kokoro/presubmit/logging-graalvm-native-presubmit.cfg b/.kokoro/presubmit/logging-graalvm-native-presubmit.cfg new file mode 100644 index 000000000000..d16598bb817f --- /dev/null +++ b/.kokoro/presubmit/logging-graalvm-native-presubmit.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.56.1" +} + +env_vars: { + key: "JOB_TYPE" + value: "graalvm-single" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/cloud-java-ci-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" +} + +env_vars: { + key: "BUILD_SUBDIR" + value: "java-logging" +} diff --git a/.kokoro/presubmit/logging-integration.cfg b/.kokoro/presubmit/logging-integration.cfg new file mode 100644 index 000000000000..30143885f2b4 --- /dev/null +++ b/.kokoro/presubmit/logging-integration.cfg @@ -0,0 +1,38 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java11" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration-single" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/cloud-java-ci-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" +} + +env_vars: { + key: "BUILD_SUBDIR" + value: "java-logging" +}