Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
3) train the submission
4) get back the predictions and the log
5) terminate the ec2 instance.
Parameters
----------
config : dict
configuration
submission_id : int
submission id
"""
conf_aws = config[AWS_CONFIG_SECTION]
instance, = launch_ec2_instances(conf_aws, nb=1)
set_submission_state(config, submission_id, 'sent_to_training')
_wait_until_ready(config, instance.id)
train_on_existing_ec2_instance(config, instance.id, submission_id)
terminate_ec2_instance(conf_aws, instance.id)
def setup(self):
"""Set up the worker.
This will launch an instance on Amazon, and copy the submission
to the instance.
"""
# sanity check for the configuration variable
for required_param in ('instance_type', 'access_key_id'):
self._check_config_name(self.config, required_param)
logger.info("Setting up AWSWorker for submission '{}'".format(
self.submission))
self.instance, = aws.launch_ec2_instances(self.config)
logger.info("Instance launched for submission '{}'".format(
self.submission))
for _ in range(5):
# try uploading the submission a few times, as this regularly fails
exit_status = aws.upload_submission(
self.config, self.instance.id, self.submission,
self.submissions_path)
if exit_status == 0:
break
else:
logger.info("Uploading submission failed, retrying ...")
if exit_status != 0:
logger.error(
'Cannot upload submission "{}"'
', an error occured'.format(self.submission))
# TODO do something with this status (no launching needs to be
event_name : str
event name
"""
conf_aws = config[AWS_CONFIG_SECTION]
secs = conf_aws[TRAIN_LOOP_INTERVAL_SECS_FIELD]
while True:
# Launch new instances for new submissions
submissions = get_submissions(config, event_name, 'new')
for submission_id, _ in submissions:
submission = get_submission_by_id(config, submission_id)
submission_name = _get_submission_folder_name(submission_id)
if submission.is_sandbox:
continue
try:
instance, = launch_ec2_instances(conf_aws, nb=1)
except botocore.exceptions.ClientError as ex:
logger.info('Exception when launching a new instance : "{}"'
.format(ex))
logger.info('Skipping...')
continue
nb_trials = 0
while nb_trials < conf_aws.get('new_instance_nb_trials', 20):
if instance.state.get('name') == 'running':
break
nb_trials += 1
time.sleep(conf_aws.get('new_instance_check_interval', 6))
_tag_instance_by_submission(conf_aws, instance.id, submission_name)
_add_or_update_tag(conf_aws, instance.id, 'train_loop', '1')
_add_or_update_tag(conf_aws, instance.id, 'event_name', event_name)
logger.info('Launched instance "{}" for submission "{}"'.format(