How to use the ramp-engine.ramp_engine.aws.api.launch_ec2_instances function in ramp-engine

To help you get started, we’ve selected a few ramp-engine examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github paris-saclay-cds / ramp-board / ramp-engine / ramp_engine / aws / aws_train.py View on Github external
3) train the submission
    4) get back the predictions and the log
    5) terminate the ec2 instance.

    Parameters
    ----------

    config : dict
        configuration

    submission_id : int
        submission id

    """
    conf_aws = config[AWS_CONFIG_SECTION]
    instance, = launch_ec2_instances(conf_aws, nb=1)
    set_submission_state(config, submission_id, 'sent_to_training')
    _wait_until_ready(config, instance.id)
    train_on_existing_ec2_instance(config, instance.id, submission_id)
    terminate_ec2_instance(conf_aws, instance.id)
github paris-saclay-cds / ramp-board / ramp-engine / ramp_engine / aws / worker.py View on Github external
def setup(self):
        """Set up the worker.

        This will launch an instance on Amazon, and copy the submission
        to the instance.
        """
        # sanity check for the configuration variable
        for required_param in ('instance_type', 'access_key_id'):
            self._check_config_name(self.config, required_param)

        logger.info("Setting up AWSWorker for submission '{}'".format(
            self.submission))
        self.instance, = aws.launch_ec2_instances(self.config)
        logger.info("Instance launched for submission '{}'".format(
            self.submission))
        for _ in range(5):
            # try uploading the submission a few times, as this regularly fails
            exit_status = aws.upload_submission(
                self.config, self.instance.id, self.submission,
                self.submissions_path)
            if exit_status == 0:
                break
            else:
                logger.info("Uploading submission failed, retrying ...")
        if exit_status != 0:
            logger.error(
                'Cannot upload submission "{}"'
                ', an error occured'.format(self.submission))
            # TODO do something with this status (no launching needs to be
github paris-saclay-cds / ramp-board / ramp-engine / ramp_engine / aws / aws_train.py View on Github external
event_name : str
        event name
    """
    conf_aws = config[AWS_CONFIG_SECTION]
    secs = conf_aws[TRAIN_LOOP_INTERVAL_SECS_FIELD]
    while True:
        # Launch new instances for new submissions
        submissions = get_submissions(config, event_name, 'new')
        for submission_id, _ in submissions:
            submission = get_submission_by_id(config, submission_id)
            submission_name = _get_submission_folder_name(submission_id)
            if submission.is_sandbox:
                continue
            try:
                instance, = launch_ec2_instances(conf_aws, nb=1)
            except botocore.exceptions.ClientError as ex:
                logger.info('Exception when launching a new instance : "{}"'
                            .format(ex))
                logger.info('Skipping...')
                continue
            nb_trials = 0
            while nb_trials < conf_aws.get('new_instance_nb_trials', 20):
                if instance.state.get('name') == 'running':
                    break
                nb_trials += 1
                time.sleep(conf_aws.get('new_instance_check_interval', 6))

            _tag_instance_by_submission(conf_aws, instance.id, submission_name)
            _add_or_update_tag(conf_aws, instance.id, 'train_loop', '1')
            _add_or_update_tag(conf_aws, instance.id, 'event_name', event_name)
            logger.info('Launched instance "{}" for submission "{}"'.format(