Skip to content

Commit 51e778e

Browse files
samcmillsamcmill
authored
Update the HDF5 and NetCDF building blocks (#503)
- Bump the default HDF5 version to 1.14.5 - Add required libxml2 dependency to NetCDF - Workaround issue when building NetCDF Fortran with the NVIDIA HPC SDK
1 parent 42e4492 commit 51e778e

File tree

5 files changed

+74
-39
lines changed

5 files changed

+74
-39
lines changed

‎docs/building_blocks.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1412,7 +1412,7 @@ non-default compilers or other toolchain options are needed. The
14121412
default is empty.
14131413

14141414
- __version__: The version of HDF5 source to download. This value is
1415-
ignored if `directory` is set. The default value is `1.12.0`.
1415+
ignored if `directory` is set. The default value is `1.14.5`.
14161416

14171417
- __with_PACKAGE[=ARG]__: Flags to control optional packages when
14181418
configuring. For instance, `with_foo=True` maps to `--with-foo`
@@ -2988,10 +2988,10 @@ directory. The default value is False.
29882988

29892989
- __ospackages__: List of OS packages to install prior to configuring
29902990
and building. For Ubuntu, the default values are
2991-
`ca-certificates`, `file`, `libcurl4-openssl-dev`, `m4`, `make`,
2992-
`wget`, and `zlib1g-dev`. For RHEL-based Linux distributions the
2993-
default values are `ca-certificates`, `file`, `libcurl-devel`
2994-
`m4`, `make`, `wget`, and `zlib-devel`.
2991+
`ca-certificates`, `file`, `libcurl4-openssl-dev`, `libxml2-dev`, `m4`,
2992+
`make`, `wget`, and `zlib1g-dev`. For RHEL-based Linux distributions the
2993+
default values are `ca-certificates`, `file`, `libcurl-devel`,
2994+
`libxml2-devel`, `m4`, `make`, `wget`, and `zlib-devel`.
29952995

29962996
- __prefix__: The top level install location. The default location is
29972997
`/usr/local/netcdf`.

‎hpccm/building_blocks/hdf5.py

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
from __future__ import unicode_literals
2222
from __future__ import print_function
2323

24+
from packaging.version import Version
25+
2426
import posixpath
2527
import re
2628
from copy import copy as _copy
@@ -92,7 +94,7 @@ class hdf5(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig):
9294
default is empty.
9395
9496
version: The version of HDF5 source to download. This value is
95-
ignored if `directory` is set. The default value is `1.12.0`.
97+
ignored if `directory` is set. The default value is `1.14.5`.
9698
9799
with_PACKAGE[=ARG]: Flags to control optional packages when
98100
configuring. For instance, `with_foo=True` maps to `--with-foo`
@@ -132,7 +134,7 @@ def __init__(self, **kwargs):
132134

133135
super(hdf5, self).__init__(**kwargs)
134136

135-
self.__baseurl = kwargs.pop('baseurl', 'https://support.hdfgroup.org/ftp/HDF5/releases')
137+
self.__baseurl = kwargs.pop('baseurl', False)
136138
self.__check = kwargs.pop('check', False)
137139
self.__configure_opts = kwargs.pop('configure_opts',
138140
['--enable-cxx',
@@ -143,7 +145,14 @@ def __init__(self, **kwargs):
143145
# without impacting the original
144146
self.__toolchain = _copy(kwargs.pop('toolchain', toolchain()))
145147
self.__runtime_ospackages = [] # Filled in by __distro()
146-
self.__version = kwargs.pop('version', '1.12.0')
148+
self.__version = kwargs.pop('version', '1.14.5')
149+
150+
if not self.__baseurl:
151+
# Download path changed with version 1.14
152+
if Version(self.__version) >= Version('1.14'):
153+
self.__baseurl = 'https://support.hdfgroup.org/releases/hdf5'
154+
else:
155+
self.__baseurl = 'https://support.hdfgroup.org/ftp/HDF5/releases'
147156

148157
# Set the Linux distribution specific parameters
149158
self.__distro()
@@ -214,11 +223,19 @@ def __download(self):
214223
# path and the tarball contains MAJOR.MINOR.REVISION, so pull
215224
# apart the full version to get the MAJOR and MINOR components.
216225
match = re.match(r'(?P<major>\d+)\.(?P<minor>\d+)', self.__version)
217-
major_minor = '{0}.{1}'.format(match.groupdict()['major'],
218-
match.groupdict()['minor'])
219-
tarball = 'hdf5-{}.tar.bz2'.format(self.__version)
220-
self.__url = '{0}/hdf5-{1}/hdf5-{2}/src/{3}'.format(
221-
self.__baseurl, major_minor, self.__version, tarball)
226+
if Version(self.__version) >= Version('1.14'):
227+
major_minor = 'v{0}_{1}'.format(match.groupdict()['major'],
228+
match.groupdict()['minor'])
229+
tarball = 'hdf5-{}.tar.gz'.format(self.__version)
230+
self.__url = '{0}/{1}/v{2}/downloads/{3}'.format(
231+
self.__baseurl, major_minor, self.__version.replace('.', '_'),
232+
tarball)
233+
else:
234+
major_minor = '{0}.{1}'.format(match.groupdict()['major'],
235+
match.groupdict()['minor'])
236+
tarball = 'hdf5-{}.tar.bz2'.format(self.__version)
237+
self.__url = '{0}/hdf5-{1}/hdf5-{2}/src/{3}'.format(
238+
self.__baseurl, major_minor, self.__version, tarball)
222239

223240
def runtime(self, _from='0'):
224241
"""Generate the set of instructions to install the runtime specific

‎hpccm/building_blocks/netcdf.py

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323

2424
from packaging.version import Version
2525
import posixpath
26+
import re
27+
from copy import copy as _copy
2628

2729
import hpccm.config
2830
import hpccm.templates.envvars
@@ -33,6 +35,7 @@
3335
from hpccm.building_blocks.packages import packages
3436
from hpccm.common import linux_distro
3537
from hpccm.primitives.comment import comment
38+
from hpccm.toolchain import toolchain
3639

3740
class netcdf(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig):
3841
"""The `netcdf` building block downloads, configures, builds, and
@@ -80,10 +83,10 @@ class netcdf(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig):
8083
8184
ospackages: List of OS packages to install prior to configuring
8285
and building. For Ubuntu, the default values are
83-
`ca-certificates`, `file`, `libcurl4-openssl-dev`, `m4`, `make`,
84-
`wget`, and `zlib1g-dev`. For RHEL-based Linux distributions the
85-
default values are `ca-certificates`, `file`, `libcurl-devel`
86-
`m4`, `make`, `wget`, and `zlib-devel`.
86+
`ca-certificates`, `file`, `libcurl4-openssl-dev`, `libxml2-dev`, `m4`,
87+
`make`, `wget`, and `zlib1g-dev`. For RHEL-based Linux distributions the
88+
default values are `ca-certificates`, `file`, `libcurl-devel`,
89+
`libxml2-devel`, `m4`, `make`, `wget`, and `zlib-devel`.
8790
8891
prefix: The top level install location. The default location is
8992
`/usr/local/netcdf`.
@@ -134,6 +137,9 @@ def __init__(self, **kwargs):
134137
self.__ospackages = kwargs.pop('ospackages', [])
135138
self.__prefix = kwargs.pop('prefix', '/usr/local/netcdf')
136139
self.__runtime_ospackages = [] # Filled in by __distro()
140+
# Create a copy of the toolchain so that it can be modified
141+
# without impacting the original
142+
self.__toolchain = _copy(kwargs.pop('toolchain', toolchain()))
137143
self.__version = kwargs.pop('version', '4.7.4')
138144
self.__version_cxx = kwargs.pop('version_cxx', '4.3.1')
139145
self.__version_fortran = kwargs.pop('version_fortran', '4.5.3')
@@ -165,6 +171,7 @@ def __init__(self, **kwargs):
165171
directory=self.__directory_c,
166172
prefix=self.__prefix,
167173
runtime_environment=self.environment_variables,
174+
toolchain=self.__toolchain,
168175
url=self.__url_c,
169176
**kwargs)]
170177

@@ -180,12 +187,19 @@ def __init__(self, **kwargs):
180187
# Checks fail when using parallel make. Disable it.
181188
parallel=1 if self.__check else '$(nproc)',
182189
prefix=self.__prefix,
190+
toolchain=self.__toolchain,
183191
url='{0}/v{1}.tar.gz'.format(self.__baseurl_cxx,
184192
self.__version_cxx),
185193
**kwargs))
186194

187195
# Setup optional Fortran build configuration
188196
if self.__fortran:
197+
# PIC workaround when using the NVIDIA compilers
198+
if self.__toolchain.FC and re.match('.*nvfortran',
199+
self.__toolchain.FC):
200+
if not self.__toolchain.FCFLAGS:
201+
self.__toolchain.FCFLAGS = '-fPIC'
202+
189203
comments.append('NetCDF Fortran version {}'.format(self.__version_fortran))
190204
self.__bb.append(generic_autotools(
191205
annotations={'version': self.__version_fortran},
@@ -196,6 +210,7 @@ def __init__(self, **kwargs):
196210
# Checks fail when using parallel make. Disable it.
197211
parallel=1 if self.__check else '$(nproc)',
198212
prefix=self.__prefix,
213+
toolchain=self.__toolchain,
199214
url='{0}/v{1}.tar.gz'.format(self.__baseurl_fortran,
200215
self.__version_fortran),
201216
**kwargs))
@@ -212,14 +227,14 @@ def __distro(self):
212227
if hpccm.config.g_linux_distro == linux_distro.UBUNTU:
213228
if not self.__ospackages:
214229
self.__ospackages = ['ca-certificates', 'file',
215-
'libcurl4-openssl-dev', 'm4', 'make',
216-
'wget', 'zlib1g-dev']
230+
'libcurl4-openssl-dev', 'libxml2-dev',
231+
'm4', 'make', 'wget', 'zlib1g-dev']
217232
self.__runtime_ospackages = ['zlib1g']
218233
elif hpccm.config.g_linux_distro == linux_distro.CENTOS:
219234
if not self.__ospackages:
220235
self.__ospackages = ['ca-certificates', 'file',
221-
'libcurl-devel', 'm4', 'make',
222-
'wget', 'zlib-devel']
236+
'libcurl-devel', 'libxml2-devel', 'm4',
237+
'make', 'wget', 'zlib-devel']
223238
if self.__check:
224239
self.__ospackages.append('diffutils')
225240
self.__runtime_ospackages = ['zlib']

‎test/test_hdf5.py

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ def setUp(self):
3636
def test_defaults_ubuntu(self):
3737
"""Default hdf5 building block"""
3838
h = hdf5()
39-
self.assertEqual(str(h),
40-
r'''# HDF5 version 1.12.0
39+
self.assertMultiLineEqual(str(h),
40+
r'''# HDF5 version 1.14.5
4141
RUN apt-get update -y && \
4242
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
4343
bzip2 \
@@ -46,12 +46,12 @@ def test_defaults_ubuntu(self):
4646
wget \
4747
zlib1g-dev && \
4848
rm -rf /var/lib/apt/lists/*
49-
RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.bz2 && \
50-
mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.12.0.tar.bz2 -C /var/tmp -j && \
51-
cd /var/tmp/hdf5-1.12.0 && ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \
49+
RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_5/downloads/hdf5-1.14.5.tar.gz && \
50+
mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.14.5.tar.gz -C /var/tmp -z && \
51+
cd /var/tmp/hdf5-1.14.5 && ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \
5252
make -j$(nproc) && \
5353
make -j$(nproc) install && \
54-
rm -rf /var/tmp/hdf5-1.12.0 /var/tmp/hdf5-1.12.0.tar.bz2
54+
rm -rf /var/tmp/hdf5-1.14.5 /var/tmp/hdf5-1.14.5.tar.gz
5555
ENV CPATH=/usr/local/hdf5/include:$CPATH \
5656
HDF5_DIR=/usr/local/hdf5 \
5757
LD_LIBRARY_PATH=/usr/local/hdf5/lib:$LD_LIBRARY_PATH \
@@ -63,21 +63,21 @@ def test_defaults_ubuntu(self):
6363
def test_defaults_centos(self):
6464
"""Default hdf5 building block"""
6565
h = hdf5()
66-
self.assertEqual(str(h),
67-
r'''# HDF5 version 1.12.0
66+
self.assertMultiLineEqual(str(h),
67+
r'''# HDF5 version 1.14.5
6868
RUN yum install -y \
6969
bzip2 \
7070
file \
7171
make \
7272
wget \
7373
zlib-devel && \
7474
rm -rf /var/cache/yum/*
75-
RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.bz2 && \
76-
mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.12.0.tar.bz2 -C /var/tmp -j && \
77-
cd /var/tmp/hdf5-1.12.0 && ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \
75+
RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_5/downloads/hdf5-1.14.5.tar.gz && \
76+
mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.14.5.tar.gz -C /var/tmp -z && \
77+
cd /var/tmp/hdf5-1.14.5 && ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \
7878
make -j$(nproc) && \
7979
make -j$(nproc) install && \
80-
rm -rf /var/tmp/hdf5-1.12.0 /var/tmp/hdf5-1.12.0.tar.bz2
80+
rm -rf /var/tmp/hdf5-1.14.5 /var/tmp/hdf5-1.14.5.tar.gz
8181
ENV CPATH=/usr/local/hdf5/include:$CPATH \
8282
HDF5_DIR=/usr/local/hdf5 \
8383
LD_LIBRARY_PATH=/usr/local/hdf5/lib:$LD_LIBRARY_PATH \
@@ -89,7 +89,7 @@ def test_defaults_centos(self):
8989
def test_ldconfig(self):
9090
"""ldconfig option"""
9191
h = hdf5(ldconfig=True, version='1.10.4')
92-
self.assertEqual(str(h),
92+
self.assertMultiLineEqual(str(h),
9393
r'''# HDF5 version 1.10.4
9494
RUN apt-get update -y && \
9595
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -117,7 +117,7 @@ def test_runtime(self):
117117
"""Runtime"""
118118
h = hdf5()
119119
r = h.runtime()
120-
self.assertEqual(r,
120+
self.assertMultiLineEqual(r,
121121
r'''# HDF5
122122
RUN apt-get update -y && \
123123
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \

‎test/test_netcdf.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,15 @@ def setUp(self):
3636
def test_defaults_ubuntu(self):
3737
"""Default netcdf building block"""
3838
n = netcdf()
39-
self.assertEqual(str(n),
39+
self.assertMultiLineEqual(str(n),
4040
r'''# NetCDF version 4.7.4, NetCDF C++ version 4.3.1, NetCDF Fortran
4141
# version 4.5.3
4242
RUN apt-get update -y && \
4343
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
4444
ca-certificates \
4545
file \
4646
libcurl4-openssl-dev \
47+
libxml2-dev \
4748
m4 \
4849
make \
4950
wget \
@@ -77,13 +78,14 @@ def test_defaults_ubuntu(self):
7778
def test_defaults_centos(self):
7879
"""Default netcdf building block"""
7980
n = netcdf()
80-
self.assertEqual(str(n),
81+
self.assertMultiLineEqual(str(n),
8182
r'''# NetCDF version 4.7.4, NetCDF C++ version 4.3.1, NetCDF Fortran
8283
# version 4.5.3
8384
RUN yum install -y \
8485
ca-certificates \
8586
file \
8687
libcurl-devel \
88+
libxml2-devel \
8789
m4 \
8890
make \
8991
wget \
@@ -118,14 +120,15 @@ def test_ldconfig(self):
118120
"""ldconfig option"""
119121
n = netcdf(ldconfig=True, version='4.6.1', version_cxx='4.3.0',
120122
version_fortran='4.4.4')
121-
self.assertEqual(str(n),
123+
self.assertMultiLineEqual(str(n),
122124
r'''# NetCDF version 4.6.1, NetCDF C++ version 4.3.0, NetCDF Fortran
123125
# version 4.4.4
124126
RUN apt-get update -y && \
125127
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
126128
ca-certificates \
127129
file \
128130
libcurl4-openssl-dev \
131+
libxml2-dev \
129132
m4 \
130133
make \
131134
wget \
@@ -162,7 +165,7 @@ def test_runtime(self):
162165
"""Runtime"""
163166
n = netcdf()
164167
r = n.runtime()
165-
self.assertEqual(r,
168+
self.assertMultiLineEqual(r,
166169
r'''# NetCDF
167170
RUN apt-get update -y && \
168171
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \

0 commit comments

Comments
 (0)