Skip to content

Commit 6218e75

Browse files
avsejavsej
committed
Update packaging scripts
* statically link BoringSSL * add script to produce "fat" binary package that includes extensions for every supported Ruby ABI
1 parent 0831b70 commit 6218e75

38 files changed

+1299
-400
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111
/.yardoc
1212
/_yardoc/
1313
/cmake-build-*/
14+
/build-*/
1415
/coverage/
1516
/doc/
17+
/ext/cache/
1618
/pkg/
1719
/test/reports/
1820
/tmp/

‎Rakefile

Lines changed: 198 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ desc "Compile binary extension"
2626
task :compile do
2727
require 'tempfile'
2828
Dir.chdir(Dir.tmpdir) do
29-
sh "ruby #{File.join(__dir__, 'ext', 'extconf.rb')}"
29+
sh "ruby '#{File.join(__dir__, 'ext', 'extconf.rb')}'"
3030
end
3131
end
3232

@@ -56,15 +56,206 @@ task :render_git_revision do
5656
`git fetch --tags >/dev/null 2>&1`
5757
`git describe --long --always HEAD`.strip
5858
end
59-
File.write(File.join(__dir__, 'ext', 'revisions.rb'), <<~REVISIONS)
60-
cmake_flags << "-DEXT_GIT_REVISION=#{library_revision}"
61-
cmake_flags << "-DCOUCHBASE_CXX_CLIENT_GIT_REVISION=#{core_revision}"
62-
cmake_flags << "-DCOUCHBASE_CXX_CLIENT_GIT_DESCRIBE=#{core_describe}"
63-
REVISIONS
59+
File.open(File.join(__dir__, "ext", "cache", "extconf_include.rb"), "a+") do |io|
60+
io.puts(<<~REVISIONS)
61+
cmake_flags << "-DEXT_GIT_REVISION=#{library_revision}"
62+
cmake_flags << "-DCOUCHBASE_CXX_CLIENT_GIT_REVISION=#{core_revision}"
63+
cmake_flags << "-DCOUCHBASE_CXX_CLIENT_GIT_DESCRIBE=#{core_describe}"
64+
REVISIONS
65+
end
66+
end
67+
68+
def which(name, extra_locations = [])
69+
ENV.fetch("PATH", "")
70+
.split(File::PATH_SEPARATOR)
71+
.prepend(*extra_locations)
72+
.select { |path| File.directory?(path) }
73+
.map { |path| [path, name].join(File::SEPARATOR) + RbConfig::CONFIG["EXEEXT"] }
74+
.find { |file| File.executable?(file) }
75+
end
76+
77+
desc "Download and cache dependencies of C++ core"
78+
task :cache_cxx_dependencies do
79+
require "tempfile"
80+
require "rubygems/package"
81+
82+
output_dir = Dir.mktmpdir("cxx_output_")
83+
output_tarball = File.join(output_dir, "cache.tar")
84+
cpm_cache_dir = Dir.mktmpdir("cxx_cache_")
85+
cxx_core_build_dir = Dir.mktmpdir("cxx_build_")
86+
cxx_core_source_dir = File.join(__dir__, "ext", "couchbase")
87+
cc = ENV.fetch("CB_CC", nil)
88+
cxx = ENV.fetch("CB_CXX", nil)
89+
ar = ENV.fetch("CB_AR", nil)
90+
91+
cmake_extra_locations = []
92+
if RUBY_PLATFORM.match?(/mswin|mingw/)
93+
cmake_extra_locations = [
94+
'C:\Program Files\CMake\bin',
95+
'C:\Program Files\Microsoft Visual Studio\2022\Professional\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin',
96+
'C:\Program Files\Microsoft Visual Studio\2019\Professional\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin',
97+
]
98+
local_app_data = ENV.fetch("LOCALAPPDATA", "#{Dir.home}\\AppData\\Local")
99+
cmake_extra_locations.unshift("#{local_app_data}\\CMake\\bin") if File.directory?(local_app_data)
100+
cc = RbConfig::CONFIG["CC"]
101+
cxx = RbConfig::CONFIG["CXX"]
102+
end
103+
cmake = which("cmake", cmake_extra_locations) || which("cmake3", cmake_extra_locations)
104+
cmake_flags = [
105+
"-S#{cxx_core_source_dir}",
106+
"-B#{cxx_core_build_dir}",
107+
"-DCOUCHBASE_CXX_CLIENT_BUILD_TESTS=OFF",
108+
"-DCOUCHBASE_CXX_CLIENT_BUILD_TOOLS=OFF",
109+
"-DCOUCHBASE_CXX_CLIENT_BUILD_DOCS=OFF",
110+
"-DCOUCHBASE_CXX_CLIENT_STATIC_BORINGSSL=ON",
111+
"-DCPM_DOWNLOAD_ALL=ON",
112+
"-DCPM_USE_NAMED_CACHE_DIRECTORIES=ON",
113+
"-DCPM_USE_LOCAL_PACKAGES=OFF",
114+
"-DCPM_SOURCE_CACHE=#{cpm_cache_dir}",
115+
]
116+
cmake_flags << "-DCMAKE_C_COMPILER=#{cc}" if cc
117+
cmake_flags << "-DCMAKE_CXX_COMPILER=#{cxx}" if cxx
118+
cmake_flags << "-DCMAKE_AR=#{ar}" if ar
119+
120+
puts("-----> run cmake to dowload all depenencies (#{cmake})")
121+
sh(cmake, *cmake_flags, verbose: true)
122+
123+
puts("-----> create archive with whitelisted sources: #{output_tarball}")
124+
File.open(output_tarball, "w+b") do |file|
125+
Gem::Package::TarWriter.new(file) do |writer|
126+
chdir(cxx_core_build_dir, verbose: true) do
127+
["mozilla-ca-bundle.sha256", "mozilla-ca-bundle.crt"].each do |path|
128+
writer.add_file(path, 0o660) { |io| io.write(File.binread(path)) }
129+
end
130+
end
131+
chdir(cpm_cache_dir, verbose: true) do
132+
third_party_sources = Dir[
133+
"cpm/*.cmake",
134+
"asio/*/LICENSE*",
135+
"asio/*/asio/COPYING",
136+
"asio/*/asio/asio/include/*.hpp",
137+
"asio/*/asio/asio/include/asio/**/*.[hi]pp",
138+
"boringssl/*/boringssl/**/*.{cc,h,c,asm,S}",
139+
"boringssl/*/boringssl/**/CMakeLists.txt",
140+
"boringssl/*/boringssl/LICENSE",
141+
"fmt/*/fmt/CMakeLists.txt",
142+
"fmt/*/fmt/ChangeLog.rst",
143+
"fmt/*/fmt/LICENSE.rst",
144+
"fmt/*/fmt/README.rst",
145+
"fmt/*/fmt/include/**/*",
146+
"fmt/*/fmt/src/**/*",
147+
"fmt/*/fmt/support/cmake/**/*",
148+
"gsl/*/gsl/CMakeLists.txt",
149+
"gsl/*/gsl/GSL.natvis",
150+
"gsl/*/gsl/LICENSE*",
151+
"gsl/*/gsl/ThirdPartyNotices.txt",
152+
"gsl/*/gsl/cmake/*",
153+
"gsl/*/gsl/include/**/*",
154+
"hdr_histogram/*/hdr_histogram/*.pc.in",
155+
"hdr_histogram/*/hdr_histogram/CMakeLists.txt",
156+
"hdr_histogram/*/hdr_histogram/COPYING.txt",
157+
"hdr_histogram/*/hdr_histogram/LICENSE.txt",
158+
"hdr_histogram/*/hdr_histogram/cmake/*",
159+
"hdr_histogram/*/hdr_histogram/config.cmake.in",
160+
"hdr_histogram/*/hdr_histogram/include/**/*",
161+
"hdr_histogram/*/hdr_histogram/src/**/*",
162+
"json/*/json/CMakeLists.txt",
163+
"json/*/json/LICENSE*",
164+
"json/*/json/external/PEGTL/.cmake/*",
165+
"json/*/json/external/PEGTL/CMakeLists.txt",
166+
"json/*/json/external/PEGTL/LICENSE*",
167+
"json/*/json/external/PEGTL/include/**/*",
168+
"json/*/json/include/**/*",
169+
"llhttp/*/llhttp/*.pc.in",
170+
"llhttp/*/llhttp/CMakeLists.txt",
171+
"llhttp/*/llhttp/LICENSE*",
172+
"llhttp/*/llhttp/include/*.h",
173+
"llhttp/*/llhttp/src/*.c",
174+
"snappy/*/snappy/CMakeLists.txt",
175+
"snappy/*/snappy/COPYING",
176+
"snappy/*/snappy/cmake/*",
177+
"snappy/*/snappy/snappy-c.{h,cc}",
178+
"snappy/*/snappy/snappy-internal.h",
179+
"snappy/*/snappy/snappy-sinksource.{h,cc}",
180+
"snappy/*/snappy/snappy-stubs-internal.{h,cc}",
181+
"snappy/*/snappy/snappy-stubs-public.h.in",
182+
"snappy/*/snappy/snappy.{h,cc}",
183+
"spdlog/*/spdlog/CMakeLists.txt",
184+
"spdlog/*/spdlog/LICENSE",
185+
"spdlog/*/spdlog/cmake/*",
186+
"spdlog/*/spdlog/include/**/*",
187+
"spdlog/*/spdlog/src/**/*",
188+
].grep_v(/crypto_test_data.cc/)
189+
190+
# we don't want to fail if git is not available
191+
cpm_cmake_path = third_party_sources.grep(/cpm.*\.cmake$/).first
192+
File.write(cpm_cmake_path, File.read(cpm_cmake_path).gsub("Git REQUIRED", "Git"))
193+
194+
third_party_sources
195+
.select { |path| File.file?(path) }
196+
.each { |path| writer.add_file(path, 0o660) { |io| io.write(File.binread(path)) } }
197+
end
198+
end
199+
end
200+
201+
rm_rf(cxx_core_build_dir, verbose: true)
202+
rm_rf(cpm_cache_dir, verbose: true)
203+
204+
untar = ["tar", "-x"]
205+
untar << "--force-local" unless RUBY_PLATFORM.match?(/darwin/)
206+
207+
puts("-----> verify that tarball works as a cache for CPM")
208+
cxx_core_build_dir = Dir.mktmpdir("cxx_build_")
209+
cpm_cache_dir = Dir.mktmpdir("cxx_cache_")
210+
chdir(cpm_cache_dir, verbose: true) do
211+
sh(*untar, "-f", output_tarball, verbose: true)
212+
end
213+
214+
cmake_flags = [
215+
"-S#{cxx_core_source_dir}",
216+
"-B#{cxx_core_build_dir}",
217+
"-DCOUCHBASE_CXX_CLIENT_BUILD_TESTS=OFF",
218+
"-DCOUCHBASE_CXX_CLIENT_BUILD_TOOLS=OFF",
219+
"-DCOUCHBASE_CXX_CLIENT_BUILD_DOCS=OFF",
220+
"-DCOUCHBASE_CXX_CLIENT_STATIC_BORINGSSL=ON",
221+
"-DCPM_DOWNLOAD_ALL=OFF",
222+
"-DCPM_USE_NAMED_CACHE_DIRECTORIES=ON",
223+
"-DCPM_USE_LOCAL_PACKAGES=OFF",
224+
"-DCPM_SOURCE_CACHE=#{cpm_cache_dir}",
225+
"-DCOUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE_ROOT=#{cpm_cache_dir}",
226+
]
227+
cmake_flags << "-DCMAKE_C_COMPILER=#{cc}" if cc
228+
cmake_flags << "-DCMAKE_CXX_COMPILER=#{cxx}" if cxx
229+
cmake_flags << "-DCMAKE_AR=#{ar}" if ar
230+
231+
sh(cmake, *cmake_flags, verbose: true)
232+
233+
rm_rf(cxx_core_build_dir, verbose: true)
234+
rm_rf(cpm_cache_dir, verbose: true)
235+
236+
cache_dir = File.join(__dir__, "ext", "cache")
237+
rm_rf(cache_dir, verbose: true)
238+
abort("unable to remove #{cache_dir}") if File.directory?(cache_dir)
239+
mkdir_p(cache_dir, verbose: true)
240+
chdir(cache_dir, verbose: true) do
241+
sh(*untar, "-f", output_tarball, verbose: true)
242+
end
243+
rm_rf(output_dir, verbose: true)
244+
245+
extconf_include = File.join(cache_dir, "extconf_include.rb")
246+
File.open(extconf_include, "w+") do |io|
247+
io.puts(<<~CACHE_FLAGS)
248+
cmake_flags << "-DCPM_DOWNLOAD_ALL=OFF"
249+
cmake_flags << "-DCPM_USE_NAMED_CACHE_DIRECTORIES=ON"
250+
cmake_flags << "-DCPM_USE_LOCAL_PACKAGES=OFF"
251+
cmake_flags << "-DCPM_SOURCE_CACHE=\#{File.expand_path('cache', __dir__)}"
252+
cmake_flags << "-DCOUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE_ROOT=\#{File.expand_path('cache', __dir__)}"
253+
CACHE_FLAGS
254+
end
64255
end
65256

66257
desc "Build the package"
67-
task :build => :render_git_revision
258+
task :build => [:cache_cxx_dependencies, :render_git_revision]
68259

69260
RuboCop::RakeTask.new
70261

‎bin/check-clang-format

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
PROJECT_ROOT="$( cd "$(dirname "$0"/..)" >/dev/null 2>&1 ; pwd -P )"
17+
PROJECT_ROOT="$( cd "$(dirname "$0")/.." >/dev/null 2>&1 ; pwd -P )"
1818

1919
CB_GIT_CLANG_FORMAT=${CB_GIT_CLANG_FORMAT:-git-clang-format}
2020
CB_CLANG_FORMAT=${CB_CLANG_FORMAT:-$(which clang-format)}

‎bin/check-clang-static-analyzer

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
PROJECT_ROOT="$( cd "$(dirname "$0"/..)" >/dev/null 2>&1 ; pwd -P )"
17+
PROJECT_ROOT="$( cd "$(dirname "$0")/.." >/dev/null 2>&1 ; pwd -P )"
1818

1919
CB_CMAKE=${CB_CMAKE:-$(which cmake)}
2020
CB_CC=${CB_CC:-$(which clang)}

‎bin/jenkins/build-environment-for-alpine.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2020-Present Couchbase, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
source /usr/local/share/chruby/chruby.sh

‎bin/jenkins/build-environment-for-amazon.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2020-Present Couchbase, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
source /usr/local/share/chruby/chruby.sh
17+
18+
export CB_CC="gcc10-cc"
19+
export CB_CXX="gcc10-c++"

‎bin/jenkins/build-environment-for-centos.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2020-Present Couchbase, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
source /usr/local/share/chruby/chruby.sh
17+
18+
CC_PREFIX=/opt/rh/devtoolset-11/root
19+
export CB_CC="${CC_PREFIX}/bin/gcc"
20+
export CB_CXX="${CC_PREFIX}/bin/g++"

‎bin/jenkins/build-environment-for-macos.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2020-Present Couchbase, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
source $(brew --prefix chruby)/share/chruby/chruby.sh
17+
18+
# MacOS/ARM has issues building BoringSSL using GCC compiler
19+
if [ "x$(uname -m)" != "xarm64" -a "x$(uname -m)" != "xx86_64" ]
20+
then
21+
for v in 12 11
22+
do
23+
CC_PREFIX=$(brew --prefix gcc@${v})
24+
if [ -d ${CC_PREFIX} ]
25+
then
26+
export CB_CC="${CC_PREFIX}/bin/gcc-${v}"
27+
export CB_CXX="${CC_PREFIX}/bin/g++-${v}"
28+
export CB_AR="${CC_PREFIX}/bin/gcc-ar-${v}"
29+
break
30+
fi
31+
done
32+
fi

‎bin/jenkins/build-environment-for-ubuntu.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2020-Present Couchbase, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
source /usr/local/share/chruby/chruby.sh

0 commit comments

Comments
 (0)