Skip to content

Commit 7745035

Browse files
author
Alvaro Muñoz
authored
Merge pull request #2 from GitHubSecurityLab/java_packs
Add combined Java QLPacks
2 parents 53aec9f + 803bbd6 commit 7745035

File tree

2,500 files changed

+64943
-27
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,500 files changed

+64943
-27
lines changed

‎.github/scripts/pr-compile.sh

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
PR_NUMBER=${1}
5+
LANGUAGE=${2}
6+
# to stop recompiling all queries if multiple files are modified
7+
LIBRARY_SCANNED=false
8+
9+
echo "[+] Compiling all queries in $LANGUAGE"
10+
gh codeql query compile --threads=0 --check-only "./$LANGUAGE/"
11+
12+
for file in $(gh pr view "$PR_NUMBER" --json files --jq '.files.[].path'); do
13+
if [[ ! -f "$file" ]]; then
14+
continue
15+
fi
16+
17+
# if the file is a query file .ql or .qll
18+
if [[ "$file" == $LANGUAGE/**.ql ]]; then
19+
echo "[+] Compiling $file (in $LANGUAGE)"
20+
21+
# compile the query
22+
gh codeql query compile --threads=0 --check-only --warnings=error "./$file"
23+
24+
# if lib folder is modified
25+
elif [[ "$file" == $LANGUAGE/lib/* ]] && [[ $LIBRARY_SCANNED == false ]]; then
26+
echo "[+] Libray changed, compiling all queries in $LANGUAGE"
27+
gh codeql query compile --threads=0 --check-only --warnings=error "./$LANGUAGE/"
28+
# set LIBRARY_SCANNED to true to prevent recompiling
29+
LIBRARY_SCANNED=true
30+
31+
fi
32+
done
33+
34+
echo "[+] Complete"

‎.github/scripts/pr-suites-packs.sh

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
PR_NUMBER=${1}
5+
LANGUAGE=${2}
6+
PACK_COMPILED=false
7+
8+
for file in $(gh pr view "$PR_NUMBER" --json files --jq '.files.[].path'); do
9+
if [[ ! -f "$file" ]]; then
10+
continue
11+
fi
12+
13+
# suite folder
14+
if [[ "$file" == $LANGUAGE/suites/**.qls ]]; then
15+
echo "[+] Compiling Suite: $file"
16+
gh codeql resolve queries "$file"
17+
18+
# qlpack file and lock file
19+
elif [[ "$file" == $LANGUAGE/qlpack.yml ]] || [[ "$file" == $LANGUAGE/codeql-pack.lock.yml ]]; then
20+
if [[ "$PACK_COMPILED" == true ]]; then
21+
continue
22+
fi
23+
echo "[+] Compiling Pack: $LANGUAGE"
24+
# install deps
25+
gh codeql pack install "$LANGUAGE"
26+
# compile / create pack
27+
gh codeql pack create "$LANGUAGE"
28+
29+
# if the version of the pack is changed, comment in the PR
30+
PUBLISHED_VERSION=$(gh api /orgs/advanced-security/packages/container/codeql-"$LANGUAGE"/versions --jq '.[0].metadata.container.tags[0]')
31+
CURRENT_VERSION=$(grep version "$LANGUAGE"/qlpack.yml | awk '{print $2}')
32+
33+
if [ "$PUBLISHED_VERSION" != "$CURRENT_VERSION" ]; then
34+
echo "[+] New version of pack detected: $PUBLISHED_VERSION (pub) != $CURRENT_VERSION (cur)"
35+
36+
comment="New version of pack \`advanced-security/codeql-$LANGUAGE\` will be created on merge: \`$PUBLISHED_VERSION\`->\`$CURRENT_VERSION\`"
37+
38+
if [[ ! $(gh pr view "$PR_NUMBER" --json comments --jq '.comments.[].body' | grep "$comment") ]]; then
39+
echo "[+] Commenting on PR"
40+
gh pr comment "$PR_NUMBER" \
41+
--body "$comment"
42+
43+
fi
44+
45+
fi
46+
47+
PACK_COMPILED=true
48+
49+
fi
50+
done
51+
52+
echo "[+] Complete"

‎.github/scripts/pr-tests.sh

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
PR_NUMBER=${1}
5+
LANGUAGE=${2}
6+
7+
if [[ ! -d ./${LANGUAGE}/test/ ]]; then
8+
echo "[!] No tests found for $LANGUAGE, skipping"
9+
exit 0
10+
fi
11+
12+
echo "[+] Compiling all queries in $LANGUAGE"
13+
gh codeql query compile --threads=0 --check-only "./$LANGUAGE/"
14+
15+
for file in $(gh pr view "$PR_NUMBER" --json files --jq '.files.[].path'); do
16+
if [[ ! -f "$file" ]]; then
17+
continue
18+
fi
19+
20+
# if a change in the test folder is detected (only for the current language)
21+
if [[ "$file" == $LANGUAGE/test/** ]]; then
22+
echo "[+] Test $file changed"
23+
TEST_DIR=$(dirname "$file")
24+
# run tests in the folder the change occured in
25+
gh codeql test run "$TEST_DIR"
26+
27+
# if the files is a query file .ql or .qll
28+
elif [[ "$file" == $LANGUAGE/**.ql ]] || [[ "$file" == $LANGUAGE/**.qll ]] ; then
29+
echo "[+] Query $file changed (in $LANGUAGE)"
30+
31+
SRC_DIR=$(realpath --relative-to="./${LANGUAGE}/src" "$file")
32+
TEST_DIR=./${LANGUAGE}/test/${SRC_DIR}
33+
34+
if [[ -d "$TEST_DIR" ]]; then
35+
echo "[+] Running tests for $file -> $TEST_DIR"
36+
gh codeql test run "$TEST_DIR"
37+
38+
else
39+
echo "[!] No tests found at $TEST_DIR"
40+
fi
41+
# if language lib folder is modified
42+
elif [[ "$file" == $LANGUAGE/lib/** ]]; then
43+
echo "[+] Library changed, running all tests in $LANGUAGE"
44+
TEST_DIR=./${LANGUAGE}/test/
45+
46+
if [[ -d "$TEST_DIR" ]]; then
47+
echo "[+] Running tests for $file -> $TEST_DIR"
48+
gh codeql test run "$TEST_DIR"
49+
else
50+
echo "[!] No tests found for $file (in $LANGUAGE)"
51+
fi
52+
53+
fi
54+
55+
done
56+
57+
echo "[+] Complete"

‎.github/workflows/build.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Build CodeQL Packs
2+
3+
on:
4+
pull_request:
5+
branches: [ main ]
6+
workflow_dispatch:
7+
8+
jobs:
9+
compile:
10+
runs-on: ubuntu-latest
11+
12+
strategy:
13+
fail-fast: false
14+
matrix:
15+
# language: [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
16+
language: [ 'java' ]
17+
18+
steps:
19+
- uses: actions/checkout@v3
20+
# with:
21+
# submodules: true
22+
23+
# Conditionally run actions based on files modified by PR, feature branch or pushed commits
24+
- uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
25+
id: changes
26+
with:
27+
filters: |
28+
src:
29+
- '${{ matrix.language }}/**'
30+
31+
- name: Install CodeQL
32+
if: steps.changes.outputs.src == 'true'
33+
env:
34+
GITHUB_TOKEN: ${{ github.token }}
35+
run: |
36+
gh extension install github/gh-codeql
37+
gh codeql pack download "codeql/${{ matrix.language }}-queries"
38+
gh codeql pack install "${{ matrix.language }}/lib"
39+
gh codeql pack install "${{ matrix.language }}/src"
40+
gh codeql pack install "${{ matrix.language }}/test"
41+
42+
- name: Compile Queries
43+
if: steps.changes.outputs.src == 'true'
44+
env:
45+
GITHUB_TOKEN: ${{ github.token }}
46+
run: |
47+
./.github/scripts/pr-compile.sh ${{ github.event.number }} ${{ matrix.language }}
48+
49+
- name: Test Queries
50+
if: steps.changes.outputs.src == 'true'
51+
env:
52+
GITHUB_TOKEN: ${{ github.token }}
53+
run: |
54+
./.github/scripts/pr-tests.sh ${{ github.event.number }} ${{ matrix.language }}
55+
56+
- name: Compile / Check Suites & Packs
57+
if: steps.changes.outputs.src == 'true'
58+
env:
59+
GITHUB_TOKEN: ${{ github.token }}
60+
run: |
61+
./.github/scripts/pr-suites-packs.sh ${{ github.event.number }} ${{ matrix.language }}
62+
63+
extensions:
64+
runs-on: ubuntu-latest
65+
66+
strategy:
67+
fail-fast: false
68+
matrix:
69+
# language: [ 'csharp', 'java', 'javascript' ]
70+
language: [ 'java' ]
71+
72+
steps:
73+
- uses: actions/checkout@v3
74+
with:
75+
submodules: true
76+
77+
- uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
78+
id: changes
79+
with:
80+
filters: |
81+
src:
82+
- '${{ matrix.language }}/ext/**'
83+
84+
- name: Install CodeQL
85+
if: steps.changes.outputs.src == 'true'
86+
env:
87+
GITHUB_TOKEN: ${{ github.token }}
88+
run: |
89+
gh extension install github/gh-codeql
90+
gh codeql pack install "${{ matrix.language }}/ext/"
91+
gh codeql pack install "${{ matrix.language }}/ext-library-sources/"
92+
gh codeql pack create "${{ matrix.language }}/ext/"
93+
gh codeql pack create "${{ matrix.language }}/ext-library-sources/"
94+

‎.github/workflows/publish.yml

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: Publish CodeQL Packs
2+
3+
on:
4+
push:
5+
branches: [main]
6+
workflow_dispatch:
7+
8+
jobs:
9+
10+
queries:
11+
runs-on: ubuntu-latest
12+
13+
permissions:
14+
contents: read
15+
packages: write
16+
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
language: ["java"]
21+
22+
steps:
23+
- uses: actions/checkout@v3
24+
25+
- name: "Check and publish codeql-LANG-queries (src) pack"
26+
env:
27+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
28+
run: |
29+
PUBLISHED_VERSION=$(gh api /orgs/githubsecuritylab/packages/container/codeql-${{ matrix.language }}-queries/versions --jq '.[0].metadata.container.tags[0]')
30+
CURRENT_VERSION=$(grep version ${{ matrix.language }}/src/qlpack.yml | awk '{print $2}')
31+
32+
if [ "$PUBLISHED_VERSION" != "$CURRENT_VERSION" ]; then
33+
gh extension install github/gh-codeql
34+
gh codeql pack install "${{ matrix.language }}/src"
35+
gh codeql pack publish "${{ matrix.language }}/src"
36+
fi
37+
38+
library:
39+
runs-on: ubuntu-latest
40+
41+
permissions:
42+
contents: read
43+
packages: write
44+
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
language: ["java"]
49+
50+
steps:
51+
- uses: actions/checkout@v3
52+
53+
- name: "Check and publish codeql-LANG-libs (lib) pack"
54+
env:
55+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
56+
run: |
57+
PUBLISHED_VERSION=$(gh api /orgs/githubsecuritylab/packages/container/codeql-${{ matrix.language }}-libs/versions --jq '.[0].metadata.container.tags[0]')
58+
CURRENT_VERSION=$(grep version ${{ matrix.language }}/lib/qlpack.yml | awk '{print $2}')
59+
60+
if [ "$PUBLISHED_VERSION" != "$CURRENT_VERSION" ]; then
61+
gh extension install github/gh-codeql
62+
gh codeql pack install "${{ matrix.language }}/lib"
63+
gh codeql pack publish "${{ matrix.language }}/lib"
64+
fi
65+
66+
extensions:
67+
runs-on: ubuntu-latest
68+
69+
strategy:
70+
fail-fast: false
71+
matrix:
72+
language: ["java"]
73+
74+
steps:
75+
- uses: actions/checkout@v3
76+
77+
- name: Check and publish codeql-LANG-extensions (ext) pack
78+
env:
79+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
80+
run: |
81+
PUBLISHED_VERSION=$(gh api /orgs/githubsecuritylab/packages/container/codeql-${{ matrix.language }}-extensions/versions --jq '.[0].metadata.container.tags[0]')
82+
CURRENT_VERSION=$(grep version ${{ matrix.language }}/ext/qlpack.yml | awk '{print $2}')
83+
84+
if [ "$PUBLISHED_VERSION" != "$CURRENT_VERSION" ]; then
85+
gh extension install github/gh-codeql
86+
gh codeql pack install "${{ matrix.language }}/ext"
87+
gh codeql pack publish "${{ matrix.language }}/ext"
88+
fi
89+
90+
library_sources_extensions:
91+
runs-on: ubuntu-latest
92+
93+
strategy:
94+
fail-fast: false
95+
matrix:
96+
language: ["java"]
97+
98+
steps:
99+
- uses: actions/checkout@v3
100+
101+
- name: Check and publish codeql-LANG-library-sources (ext-library-sources) pack
102+
env:
103+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
104+
run: |
105+
PUBLISHED_VERSION=$(gh api /orgs/githubsecuritylab/packages/container/codeql-${{ matrix.language }}-library-sources/versions --jq '.[0].metadata.container.tags[0]')
106+
CURRENT_VERSION=$(grep version ${{ matrix.language }}/ext-library-sources/qlpack.yml | awk '{print $2}')
107+
108+
if [ "$PUBLISHED_VERSION" != "$CURRENT_VERSION" ]; then
109+
gh extension install github/gh-codeql
110+
gh codeql pack install "${{ matrix.language }}/ext-library-sources"
111+
gh codeql pack publish "${{ matrix.language }}/ext-library-sources"
112+
fi

‎codeql-workspace.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
provide:
2+
- java/**/qlpack.yml

‎java/ext-library-sources/codeql-pack.lock.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
lockVersion: 1.0.0
3+
dependencies: {}
4+
compiled: false

‎java/ext/library_sources/generated/.gitkeep renamed to ‎java/ext-library-sources/models/generated/.gitkeep

File renamed without changes.

‎java/ext-library-sources/models/generated/alfio.controller.form.model.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/java-all
4+
extensible: sourceModel
5+
data:
6+
- ["alfio.controller.form", "ReservationForm", True, "setPromoCode", "(String)", "", "Parameter[0]", "remote", "manual"]

‎java/ext-library-sources/models/generated/alfio.manager.system.model.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/java-all
4+
extensible: sourceModel
5+
data:
6+
- ["alfio.manager.system", "ExternalConfiguration", True, "getSingle", "(String)", "", "Parameter[0]", "remote", "manual"]

‎java/ext-library-sources/models/generated/alluxio.collections.model.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/java-all
4+
extensible: sourceModel
5+
data:
6+
- ["alluxio.collections", "LockPool<String>", True, "get", "(String,LockMode)", "", "Parameter[0]", "remote", "manual"]
7+
- ["alluxio.collections", "TwoKeyConcurrentMap<Long,String,Long,SortedMap<String,Long>>", True, "addInnerValue", "(Long,String,Long)", "", "Parameter[1]", "remote", "manual"]
8+
- ["alluxio.collections", "TwoKeyConcurrentMap<Long,String,Long,SortedMap<String,Long>>", True, "removeInnerValue", "(Long,String)", "", "Parameter[1]", "remote", "manual"]

‎java/ext-library-sources/models/generated/alluxio.job.wire.model.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/java-all
4+
extensible: sourceModel
5+
data:
6+
- ["alluxio.job.wire", "Status", False, "valueOf", "(String)", "", "Parameter[0]", "remote", "manual"]

0 commit comments

Comments
 (0)