How to use easybuild - 10 common examples

self.assertEqual(len(downloads), 1)

        # non-existing files result in None return value
        self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None)

        # install broken proxy handler for opening local files
        # this should make urlopen use this broken proxy for downloading from a file:// URL
        proxy_handler = std_urllib.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir})
        std_urllib.install_opener(std_urllib.build_opener(proxy_handler))

        # downloading over a broken proxy results in None return value (failed download)
        # this tests whether proxies are taken into account by download_file
        self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails")

        # modify existing download so we can verify re-download
        ft.write_file(target_location, '')

        # restore a working file handler, and retest download of local file
        std_urllib.install_opener(std_urllib.build_opener(std_urllib.FileHandler()))
        res = ft.download_file(fn, source_url, target_location)
        self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy")

        # existing file was re-downloaded, so a backup should have been created of the existing file
        downloads = glob.glob(target_location + '*')
        self.assertEqual(len(downloads), 2)
        backup = [d for d in downloads if os.path.basename(d) != fn][0]
        self.assertEqual(ft.read_file(backup), '')
        self.assertEqual(ft.compute_checksum(target_location), ft.compute_checksum(os.path.join(toy_source_dir, fn)))

        # make sure specified timeout is parsed correctly (as a float, not a string)
        opts = init_config(args=['--download-timeout=5.3'])
        init_config(build_options={'download_timeout': opts.download_timeout})

:return: total memory as an integer, specifically a number of megabytes
    """
    memtotal = None
    os_type = get_os_type()

    if os_type == LINUX and is_readable(PROC_MEMINFO_FP):
        _log.debug("Trying to determine total memory size on Linux via %s", PROC_MEMINFO_FP)
        meminfo = read_file(PROC_MEMINFO_FP)
        mem_mo = re.match(r'^MemTotal:\s*(\d+)\s*kB', meminfo, re.M)
        if mem_mo:
            memtotal = int(mem_mo.group(1)) // 1024

    elif os_type == DARWIN:
        cmd = "sysctl -n hw.memsize"
        _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd)
        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
        if ec == 0:
            memtotal = int(out.strip()) // (1024**2)

    if memtotal is None:
        memtotal = UNKNOWN
        _log.warning("Failed to determine total memory, returning %s", memtotal)

    return memtotal

def install_step(self):
        """Copy all files in build directory to the install directory"""
        # can't use shutil.copytree because that doesn't allow the target directory to exist already
        run_cmd("cp -a %s/* %s" % (self.cfg['start_dir'], self.installdir))

def run_test(custom=None, extra_params=[], fmt=None):
            """Inner function to run actual test in current setting."""

            fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
            os.close(fd)

            avail_args = [
                '-a',
                '--avail-easyconfig-params',
            ]
            for avail_arg in avail_args:

                # clear log
                write_file(self.logfile, '')

                args = [
                    '--unittest-file=%s' % self.logfile,
                    avail_arg,
                ]
                if fmt is not None:
                    args.append(fmt)
                if custom is not None:
                    args.extend(['-e', custom])

                outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True)
                logtxt = read_file(self.logfile)

                # check whether all parameter types are listed
                par_types = [BUILD, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT,
                             LICENSE, MANDATORY, MODULES, OTHER, TOOLCHAIN]

'description = "%s"' % descr,
            "toolchain = {'name': 'dummy', 'version': 'dummy'}",
            "dependencies = [('GCC', '6.4.0-2.28'), ('toy', '0.0-deps')]",
            "builddependencies = [('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]",
            # hidden deps must be included in list of (build)deps
            "hiddendependencies = [('toy', '0.0-deps'), ('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]",
            "modextravars = %s" % str(modextravars),
            "modextrapaths = %s" % str(modextrapaths),
        ])

        test_dir = os.path.dirname(os.path.abspath(__file__))
        os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')

        # test if module is generated correctly
        self.writeEC()
        ec = EasyConfig(self.eb_file)
        eb = EasyBlock(ec)
        eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
        eb.check_readiness_step()
        eb.make_builddir()
        eb.prepare_step()

        modpath = os.path.join(eb.make_module_step(), name, version)
        if get_module_syntax() == 'Lua':
            modpath += '.lua'
        self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)

        # verify contents of module
        txt = read_file(modpath)
        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0]))
            self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M))

# initialize configuration so config.get_modules_tool function works
    eb_go = eboptions.parse_options(args=args, with_include=with_include)
    config.init(eb_go.options, eb_go.get_options_by_section('config'))

    # initialize build options
    if build_options is None:
        build_options = {}

    default_build_options = {
        'extended_dry_run': False,
        'external_modules_metadata': ConfigObj(),
        'local_var_naming_check': 'error',
        'silence_deprecation_warnings': eb_go.options.silence_deprecation_warnings,
        'suffix_modules_path': GENERAL_CLASS,
        'valid_module_classes': module_classes(),
        'valid_stops': [x[0] for x in EasyBlock.get_steps()],
    }
    for key in default_build_options:
        if key not in build_options:
            build_options[key] = default_build_options[key]

    config.init_build_options(build_options=build_options)

    return eb_go.options

"""Baseclass for easyconfig testcases."""

    # initialize configuration (required for e.g. default modules_tool setting)
    eb_go = eboptions.parse_options()
    config.init(eb_go.options, eb_go.get_options_by_section('config'))
    build_options = {
        'check_osdeps': False,
        'external_modules_metadata': {},
        'force': True,
        'local_var_naming_check': 'error',
        'optarch': 'test',
        'robot_path': get_paths_for("easyconfigs")[0],
        'silent': True,
        'suffix_modules_path': GENERAL_CLASS,
        'valid_module_classes': config.module_classes(),
        'valid_stops': [x[0] for x in EasyBlock.get_steps()],
    }
    config.init_build_options(build_options=build_options)
    set_tmpdir()
    del eb_go

    # put dummy 'craype-test' module in place, which is required for parsing easyconfigs using Cray* toolchains
    TMPDIR = tempfile.mkdtemp()
    os.environ['MODULEPATH'] = TMPDIR
    write_file(os.path.join(TMPDIR, 'craype-test'), '#%Module\n')

    log = fancylogger.getLogger("EasyConfigTest", fname=False)

    # make sure a logger is present for main
    eb_main._log = log
    ordered_specs = None
    parsed_easyconfigs = []

"toolchain = {'name': 'dummy', 'version': 'dummy'}",
            "dependencies = [('GCC', '6.4.0-2.28'), ('toy', '0.0-deps')]",
            "builddependencies = [('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]",
            # hidden deps must be included in list of (build)deps
            "hiddendependencies = [('toy', '0.0-deps'), ('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]",
            "modextravars = %s" % str(modextravars),
            "modextrapaths = %s" % str(modextrapaths),
        ])

        test_dir = os.path.dirname(os.path.abspath(__file__))
        os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules')

        # test if module is generated correctly
        self.writeEC()
        ec = EasyConfig(self.eb_file)
        eb = EasyBlock(ec)
        eb.installdir = os.path.join(config.install_path(), 'pi', '3.14')
        eb.check_readiness_step()
        eb.make_builddir()
        eb.prepare_step()

        modpath = os.path.join(eb.make_module_step(), name, version)
        if get_module_syntax() == 'Lua':
            modpath += '.lua'
        self.assertTrue(os.path.exists(modpath), "%s exists" % modpath)

        # verify contents of module
        txt = read_file(modpath)
        if get_module_syntax() == 'Tcl':
            self.assertTrue(re.search(r"^#%Module", txt.split('\n')[0]))
            self.assertTrue(re.search(r"^conflict\s+%s$" % name, txt, re.M))

self.assertTrue('intel/2018a' in full_mod_names)

        # here we have included a dependency in the easyconfig list
        easyconfig['full_mod_name'] = 'gzip/1.4'

        ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)]
        build_options.update({'robot_path': None})
        init_config(build_options=build_options)
        res = resolve_dependencies(ecs, self.modtool)
        # all dependencies should be resolved
        self.assertEqual(0, sum(len(ec['dependencies']) for ec in res))

        # this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled
        ecs = [deepcopy(easyconfig_dep)]
        msg = "Missing dependencies"
        self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs, self.modtool)

        # test if dependencies of an automatically found file are also loaded
        easyconfig_dep['dependencies'] = [{
            'name': 'gzip',
            'version': '1.4',
            'versionsuffix': '',
            'toolchain': {'name': 'GCC', 'version': '4.6.3'},
            'dummy': True,
            'hidden': False,
        }]
        ecs = [deepcopy(easyconfig_dep)]
        build_options.update({'robot_path': base_easyconfig_dir})
        init_config(build_options=build_options)
        res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool)

        # GCC should be first (required by gzip dependency)

elif get_module_syntax() == 'Lua':
            modules_header_txt = '\n'.join([
                "-- test header",
                'setenv("SITE_SPECIFIC_HEADER_ENV_VAR", "foo")',
            ])
            modules_footer_txt = '\n'.join([
                "-- test footer",
                'setenv("SITE_SPECIFIC_FOOTER_ENV_VAR", "bar")',
            ])
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # dump header/footer text to file
        handle, modules_footer = tempfile.mkstemp(prefix='modules-footer-')
        os.close(handle)
        write_file(modules_footer, modules_footer_txt)
        handle, modules_header = tempfile.mkstemp(prefix='modules-header-')
        os.close(handle)
        write_file(modules_header, modules_header_txt)

        # use toy-0.0.eb easyconfig file that comes with the tests
        eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb')

        # check log message with --skip for existing module
        args = [
            eb_file,
            '--sourcepath=%s' % self.test_sourcepath,
            '--buildpath=%s' % self.test_buildpath,
            '--installpath=%s' % self.test_installpath,
            '--debug',
            '--force',
            '--modules-header=%s' % modules_header,