Skip to content

Commit 4a511d9

Browse files
pietroalbinipietroalbini
committed
ci: run all the prepare scripts into a single step
1 parent b3fa3bc commit 4a511d9

File tree

2 files changed

+95
-93
lines changed

2 files changed

+95
-93
lines changed

‎src/ci/azure-pipelines/steps/run.yml

Lines changed: 2 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -48,103 +48,12 @@ steps:
4848
- bash: python src/ci/cpu-usage-over-time.py &> cpu-usage.csv &
4949
displayName: "Collect CPU-usage statistics in the background"
5050

51-
- bash: src/ci/scripts/dump-environment.sh
52-
displayName: Show the current environment
53-
54-
- bash: src/ci/scripts/install-sccache.sh
55-
env:
56-
AGENT_OS: $(Agent.OS)
57-
displayName: Install sccache
58-
condition: and(succeeded(), not(variables.SKIP_JOB))
59-
60-
- bash: src/ci/scripts/install-clang.sh
61-
env:
62-
AGENT_OS: $(Agent.OS)
63-
displayName: Install clang
64-
condition: and(succeeded(), not(variables.SKIP_JOB))
65-
66-
- bash: src/ci/scripts/switch-xcode.sh
67-
env:
68-
AGENT_OS: $(Agent.OS)
69-
displayName: Switch to Xcode 9.3
70-
condition: and(succeeded(), not(variables.SKIP_JOB))
71-
72-
- bash: src/ci/scripts/install-wix.sh
73-
env:
74-
AGENT_OS: $(Agent.OS)
75-
displayName: Install wix
76-
condition: and(succeeded(), not(variables.SKIP_JOB))
77-
78-
- bash: src/ci/scripts/install-innosetup.sh
79-
env:
80-
AGENT_OS: $(Agent.OS)
81-
displayName: Install InnoSetup
82-
condition: and(succeeded(), not(variables.SKIP_JOB))
83-
84-
- bash: src/ci/scripts/windows-symlink-build-dir.sh
85-
env:
86-
AGENT_OS: $(Agent.OS)
87-
displayName: Ensure the build happens on C:\ instead of D:\
88-
condition: and(succeeded(), not(variables.SKIP_JOB))
89-
90-
- bash: src/ci/scripts/disable-git-crlf-conversion.sh
91-
displayName: "Disable git automatic line ending conversion (on C:/)"
92-
condition: and(succeeded(), not(variables.SKIP_JOB))
93-
94-
- bash: src/ci/scripts/install-msys2.sh
51+
- bash: src/ci/prepare.py
9552
env:
9653
AGENT_OS: $(Agent.OS)
9754
SYSTEM_WORKFOLDER: $(System.Workfolder)
98-
displayName: Install msys2
99-
condition: and(succeeded(), not(variables.SKIP_JOB))
100-
101-
- bash: src/ci/scripts/install-mingw.sh
102-
env:
103-
AGENT_OS: $(Agent.OS)
104-
SYSTEM_WORKFOLDER: $(System.Workfolder)
105-
displayName: Install MinGW
106-
condition: and(succeeded(), not(variables.SKIP_JOB))
107-
108-
- bash: src/ci/scripts/install-ninja.sh
109-
env:
110-
AGENT_OS: $(Agent.OS)
111-
displayName: Install ninja
112-
condition: and(succeeded(), not(variables.SKIP_JOB))
113-
114-
- bash: src/ci/scripts/enable-docker-ipv6.sh
115-
env:
116-
AGENT_OS: $(Agent.OS)
117-
displayName: Enable IPv6 on Docker
118-
condition: and(succeeded(), not(variables.SKIP_JOB))
119-
120-
# Disable automatic line ending conversion (again). On Windows, when we're
121-
# installing dependencies, something switches the git configuration directory or
122-
# re-enables autocrlf. We've not tracked down the exact cause -- and there may
123-
# be multiple -- but this should ensure submodules are checked out with the
124-
# appropriate line endings.
125-
- bash: src/ci/scripts/disable-git-crlf-conversion.sh
126-
displayName: Disable git automatic line ending conversion
127-
condition: and(succeeded(), not(variables.SKIP_JOB))
128-
129-
- bash: src/ci/scripts/checkout-submodules.sh
130-
env:
131-
AGENT_OS: $(Agent.OS)
132-
displayName: Checkout submodules
133-
condition: and(succeeded(), not(variables.SKIP_JOB))
134-
135-
- bash: src/ci/scripts/verify-line-endings.sh
136-
env:
137-
AGENT_OS: $(Agent.OS)
138-
displayName: Verify line endings
139-
condition: and(succeeded(), not(variables.SKIP_JOB))
140-
141-
# Ensure the `aws` CLI is installed so we can deploy later on, cache docker
142-
# images, etc.
143-
- bash: src/ci/scripts/install-awscli.sh
144-
env:
145-
AGENT_OS: $(Agent.OS)
55+
displayName: Prepare the CI environment
14656
condition: and(succeeded(), not(variables.SKIP_JOB))
147-
displayName: Install awscli
14857

14958
# Configure our CI_JOB_NAME variable which log analyzers can use for the main
15059
# step to see what's going on.

‎src/ci/prepare.py

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/usr/bin/env python3
2+
3+
import io
4+
import os
5+
import re
6+
import subprocess
7+
8+
def run(b):
9+
b.step("dump-environment.sh", "Show the current environment")
10+
b.step("install-sccache.sh", "Install sccache")
11+
b.step("install-clang.sh", "Install clang")
12+
b.step("switch-xcode.sh", "Switch to Xcode 9.3")
13+
b.step("install-wix.sh", "Install WIX")
14+
b.step("install-innosetup.sh", "Install InnoSetup")
15+
b.step(
16+
"windows-symlink-build-dir.sh",
17+
"Ensure the build happens on C:\\ instead of D:\\",
18+
)
19+
b.step(
20+
"disable-git-crlf-conversion.sh",
21+
"Disable git automatic line ending conversion (on C:\\)",
22+
)
23+
b.step("install-msys2.sh", "Install msys2")
24+
b.step("install-mingw.sh", "Install MinGW")
25+
b.step("install-ninja.sh", "Install ninja")
26+
b.step("enable-docker-ipv6.sh", "Enable IPv6 on Docker")
27+
28+
# Disable automatic line ending conversion (again). On Windows, when we're
29+
# installing dependencies, something switches the git configuration directory or
30+
# re-enables autocrlf. We've not tracked down the exact cause -- and there may
31+
# be multiple -- but this should ensure submodules are checked out with the
32+
# appropriate line endings.
33+
b.step(
34+
"disable-git-crlf-conversion.sh",
35+
"Disable git automatic line ending conversion",
36+
)
37+
38+
b.step("checkout-submodules.sh", "Checkout submodules")
39+
b.step("verify-line-endings.sh", "Verify line endings")
40+
b.step("install-awscli.sh", "Install awscli")
41+
42+
LOG_COMMAND_ADD_PATH_RE = re.compile(r"^##vso\[task\.prependpath\](.*)$")
43+
LOG_COMMAND_SET_ENV_RE = re.compile(
44+
r"^##vso\[task\.setvariable variable=([0-9A-Za-z_]+)\](.*)$"
45+
)
46+
47+
class Build:
48+
def __init__(self):
49+
self.env = dict(os.environ)
50+
51+
def step(self, script, name):
52+
print("==== %s ====" % name)
53+
54+
# While the scripts can be executed mostly as-is, they emit log
55+
# commands understood by the CI agent. Some of those commands (such as
56+
# setting new environment variables and adding items to the path) need
57+
# to be executed in order for the following scripts to succeed, so we
58+
# read the process output line by line and emulate them.
59+
path = os.path.join(os.path.dirname(__file__), "scripts", script)
60+
proc = subprocess.Popen(
61+
[path],
62+
env=self.env,
63+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
64+
)
65+
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"):
66+
self.handle_log_commands(line)
67+
print(line, end="")
68+
proc.wait()
69+
70+
if proc.returncode != 0:
71+
print("==== failure: %s ====" % name)
72+
print("exit code: %s" % proc.returncode)
73+
exit(proc.returncode)
74+
else:
75+
print("==== success: %s ====" % name)
76+
print()
77+
78+
def handle_log_commands(self, line):
79+
if not line.startswith("##vso["):
80+
return
81+
82+
add_path = LOG_COMMAND_ADD_PATH_RE.match(line)
83+
set_env = LOG_COMMAND_SET_ENV_RE.match(line)
84+
if add_path is not None:
85+
self.env["PATH"] = add_path.group(1) + ":" + self.env["PATH"]
86+
elif set_env is not None:
87+
self.env[set_env.group(1)] = set_env.group(2)
88+
else:
89+
print("ERROR: src/ci/prepare.py: unsupported log command: " + line)
90+
exit(1)
91+
92+
if __name__ == "__main__":
93+
run(Build())

0 commit comments

Comments
 (0)