Skip to content

Commit c5e5b98

Browse files
mbaludambaluda
authored
Merge pull request #1186 from github/mbaluda-caching
Add caching to query compilation and dependencies installation
2 parents 3baed90 + 41941ca commit c5e5b98

4 files changed

Lines changed: 64 additions & 17 deletions

File tree

‎.github/actions/install-codeql-packs/action.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ inputs:
77
The path to the CodeQL CLI directory.
88
required: false
99

10+
common_caches:
11+
description: |
12+
The path to the CodeQL common caches directory.
13+
required: false
14+
1015
mode:
1116
description: |
1217
The `--mode` option to `codeql pack install`.
@@ -22,4 +27,6 @@ runs:
2227
CODEQL_CLI: ${{ inputs.cli_path }}
2328
run: |
2429
PATH=$PATH:$CODEQL_CLI
25-
python scripts/install-packs.py --mode ${{ inputs.mode }}
30+
python scripts/install-packs.py \
31+
--mode ${{ inputs.mode }} \
32+
--common-caches "${{ inputs.common_caches }}"

‎.github/workflows/code-scanning-pack-gen.yml‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ env:
2020
XARGS_MAX_PROCS: 4
2121

2222
jobs:
23-
2423
prepare-code-scanning-pack-matrix:
2524
name: Prepare CodeQL Code Scanning pack matrix
2625
runs-on: ubuntu-22.04
@@ -62,10 +61,19 @@ jobs:
6261
codeql-home: ${{ github.workspace }}/codeql_home
6362
add-to-path: false
6463

64+
- name: Cache queries compilation
65+
id: cache-queries-compilation
66+
uses: actions/cache/restore@v6
67+
with:
68+
path: ${{ github.workspace }}/codeql_cache
69+
key: ${{ runner.os }}-codeql-compilation-code-scanning-pack-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}-${{ github.run_id }}
70+
restore-keys: ${{ runner.os }}-codeql-compilation-code-scanning-pack-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}
71+
6572
- name: Install CodeQL packs
6673
uses: ./.github/actions/install-codeql-packs
6774
with:
6875
cli_path: ${{ github.workspace }}/codeql_home/codeql
76+
common_caches: ${{ github.workspace }}/codeql_cache
6977

7078
- name: Determine ref for external help files
7179
id: determine-ref
@@ -105,11 +113,22 @@ jobs:
105113
PATH=$PATH:$CODEQL_HOME/codeql
106114
# Precompile all queries, and use a compilation cache larger than default
107115
# to ensure we cache all the queries for later steps
108-
codeql query compile --precompile --threads 0 --compilation-cache-size=1024 cpp c
116+
codeql query compile \
117+
--common-caches=${{ github.workspace }}/codeql_cache \
118+
--precompile \
119+
--threads 0 \
120+
--compilation-cache-size=1024 \
121+
cpp c
109122
110123
cd ..
111124
zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/schemas
112125
126+
- name: Save queries compilation cache
127+
uses: actions/cache/save@v6
128+
with:
129+
path: ${{ github.workspace }}/codeql_cache
130+
key: ${{ steps.cache-queries-compilation.outputs.cache-primary-key }}
131+
113132
- name: Upload GHAS Query Pack
114133
uses: actions/upload-artifact@v7
115134
with:
@@ -119,20 +138,21 @@ jobs:
119138
- name: Create qlpack bundles
120139
env:
121140
CODEQL_HOME: ${{ github.workspace }}/codeql_home
141+
CODEQL_CACHE: ${{ github.workspace }}/codeql_cache
122142
run: |
123143
PATH=$PATH:$CODEQL_HOME/codeql
124144
125-
codeql pack bundle --output=common-cpp-coding-standards.tgz cpp/common/src
126-
codeql pack bundle --output=common-c-coding-standards.tgz c/common/src
127-
codeql pack bundle --output=misra-c-coding-standards.tgz c/misra/src
128-
codeql pack bundle --output=cert-c-coding-standards.tgz c/cert/src
129-
codeql pack bundle --output=cert-cpp-coding-standards.tgz cpp/cert/src
130-
codeql pack bundle --output=autosar-cpp-coding-standards.tgz cpp/autosar/src
131-
codeql pack bundle --output=misra-cpp-coding-standards.tgz cpp/misra/src
132-
codeql pack bundle --output=report-coding-standards.tgz cpp/report/src
145+
codeql pack bundle --common-caches="$CODEQL_CACHE" --output=common-cpp-coding-standards.tgz cpp/common/src
146+
codeql pack bundle --common-caches="$CODEQL_CACHE" --output=common-c-coding-standards.tgz c/common/src
147+
codeql pack bundle --common-caches="$CODEQL_CACHE" --output=misra-c-coding-standards.tgz c/misra/src
148+
codeql pack bundle --common-caches="$CODEQL_CACHE" --output=cert-c-coding-standards.tgz c/cert/src
149+
codeql pack bundle --common-caches="$CODEQL_CACHE" --output=cert-cpp-coding-standards.tgz cpp/cert/src
150+
codeql pack bundle --common-caches="$CODEQL_CACHE" --output=autosar-cpp-coding-standards.tgz cpp/autosar/src
151+
codeql pack bundle --common-caches="$CODEQL_CACHE" --output=misra-cpp-coding-standards.tgz cpp/misra/src
152+
codeql pack bundle --common-caches="$CODEQL_CACHE" --output=report-coding-standards.tgz cpp/report/src
133153
134154
- name: Upload qlpack bundles
135155
uses: actions/upload-artifact@v7
136156
with:
137157
name: coding-standards-codeql-packs
138-
path: '*-coding-standards.tgz'
158+
path: "*-coding-standards.tgz"

‎.github/workflows/codeql_unit_tests.yml‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ on:
1717
- "rc/**"
1818

1919
jobs:
20-
2120
prepare-unit-test-matrix:
2221
name: Prepare CodeQL unit test matrix
2322
runs-on: ubuntu-22.04
@@ -59,7 +58,7 @@ jobs:
5958

6059
- name: Cache CodeQL
6160
id: cache-codeql
62-
uses: actions/cache@v5
61+
uses: actions/cache@v6
6362
with:
6463
# A list of files, directories, and wildcard patterns to cache and restore
6564
path: ${{github.workspace}}/codeql_home
@@ -75,16 +74,34 @@ jobs:
7574
codeql-home: ${{ github.workspace }}/codeql_home
7675
add-to-path: false
7776

77+
- name: Cache queries compilation
78+
id: cache-queries-compilation
79+
uses: actions/cache/restore@v6
80+
with:
81+
path: ${{ github.workspace }}/codeql_cache
82+
key: ${{ runner.os }}-codeql-compilation-${{ matrix.language }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }}-${{ github.run_id }}
83+
restore-keys: ${{ runner.os }}-codeql-compilation-${{ matrix.language }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }}
84+
7885
- name: Install CodeQL packs
7986
uses: ./.github/actions/install-codeql-packs
8087
with:
8188
cli_path: ${{ github.workspace }}/codeql_home/codeql
89+
common_caches: ${{ github.workspace }}/codeql_cache
8290

8391
- name: Pre-Compile Queries
8492
id: pre-compile-queries
8593
run: |
86-
${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 ${{ matrix.language }}
87-
94+
${{ github.workspace }}/codeql_home/codeql/codeql query compile \
95+
--common-caches=${{ github.workspace }}/codeql_cache \
96+
--compilation-cache-size=1024 \
97+
--threads 0 \
98+
${{ matrix.language }}
99+
100+
- name: Save queries compilation cache
101+
uses: actions/cache/save@v6
102+
with:
103+
path: ${{ github.workspace }}/codeql_cache
104+
key: ${{ steps.cache-queries-compilation.outputs.cache-primary-key }}
88105

89106
- name: Run test suites
90107
id: run-test-suites
@@ -135,7 +152,7 @@ jobs:
135152
os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
136153
test_report_file = open(test_report_path, 'w')
137154
files_to_close.append(test_report_file)
138-
procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
155+
procs.append(subprocess.Popen([codeql_bin, "test", "run", "--common-caches=${{ github.workspace }}/codeql_cache", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
139156
140157
for p in procs:
141158
_, err = p.communicate()

‎scripts/install-packs.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
parser = argparse.ArgumentParser(description="Install CodeQL library pack dependencies.")
77
parser.add_argument('--mode', required=False, choices=['use-lock', 'update', 'verify', 'no-lock'], default="use-lock", help="Installation mode, identical to the `--mode` argument to `codeql pack install`")
88
parser.add_argument('--codeql', required=False, default='codeql', help="Path to the `codeql` executable.")
9+
parser.add_argument('--common-caches', required=False, help="Path to the CodeQL common caches directory.")
910
args = parser.parse_args()
1011

1112
# Find the root of the repo
@@ -19,5 +20,7 @@
1920
pack_path = os.path.join(root, pack)
2021
# Run `codeql pack install` to install dependencies.
2122
command = [args.codeql, 'pack', 'install', '--allow-prerelease', '--mode', args.mode, pack_path]
23+
if args.common_caches:
24+
command.insert(3, f'--common-caches={args.common_caches}')
2225
print(f'Running `{" ".join(command)}`')
2326
subprocess.check_call(command)

0 commit comments

Comments
 (0)