Source code for

# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
# ReFrame Project Developers. See the top-level LICENSE file for details.
# SPDX-License-Identifier: BSD-3-Clause

import reframe as rfm
import reframe.utility.sanity as sn

[docs]@rfm.simple_test class pytorch_cnn_check(rfm.RunOnlyRegressionTest, pin_prefix=True): '''Run a synthetic CNN benchmark with PyTorch and Horovod. PyTorch is a Python package that provides tensor computation like NumPy with strong GPU acceleration and deep neural networks built on a tape-based autograd system. For more information, refer to `<>`__. Horovod is a distributed deep learning training framework for TensorFlow, Keras, PyTorch, and Apache MXNet. The goal of Horovod is to make distributed deep learning fast and easy to use. For more information refer to `<>`__. This test runs the Horovod ```` example, checks its sanity and extracts the GPU performance. ''' #: The version of Horovod to use. #: #: :type: :class:`str` #: :default: ``'v0.21.0'`` benchmark_version = variable(str, value='v0.21.0') #: The name of the model to use for this benchmark. #: #: :type: :class:`str` #: :default: ``'inception_v3'`` model = variable(str, value='inception_v3') #: The size of the batch used during the learning of models. #: #: :type: :class:`int` #: :default: ``64`` batch_size = variable(int, value=64) #: The number of iterations. #: #: :type: :class:`int` #: :default: ``5`` num_iters = variable(int, value=5) #: The number of batches per iteration. #: #: :type: :class:`int` #: :default: ``5`` num_batches_per_iter = variable(int, value=5) #: The number of warmup batches #: #: :type: :class:`int` #: :default: ``5`` num_warmup_batches = variable(int, value=5) executable = 'python' tags = {'ml', 'cnn', 'horovod'} @run_after('init') def update_descr(self): # Get the python script self.descr = (f'Distributed CNN training with PyTorch and Horovod ' f'(model: {self.model})') @run_before('run') def prepare_run(self): script = self.executable.split()[1] self.prerun_cmds = [ f'curl -LJO{self.benchmark_version}/examples/pytorch/{script}', # noqa: E501 ] if self.model == 'inception_v3': self.prerun_cmds += [ f'sed -i "s/output = model/output, aux = model/g" {script}', f'sed -i "s/224, 224/299, 299/g" {script}' ] self.executable_opts = [ f'--model {self.model}', f'--batch-size {self.batch_size}', f'--num-iters {self.num_iters}', f'--num-batches-per-iter {self.num_batches_per_iter}', f'--num-warmup-batches {self.num_warmup_batches}' ]
[docs] @performance_function('images/s') def throughput_iteration(self): '''The average GPU throughput per iteration in ``images/s``.''' return sn.avg( sn.extractall(r'Img/sec per GPU: (\S+) \S+', self.stdout, 1, float) )
[docs] @performance_function('images/s') def throughput_total(self): '''The total GPU throughput of the benchmark in ``images/s``.''' return sn.extractsingle( rf'Total img/sec on {self.num_tasks} GPU\(s\): (\S+) \S+', self.stdout, 1, float )
@sanity_function def validate_run(self): return sn.all([ sn.assert_found(rf'Model: {self.model}', self.stdout), sn.assert_found(rf'Batch size: {self.batch_size}', self.stdout) ])