Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
logger.info('Download max ram usage info of "{}"'.format(label))
download_mprof_data(conf_aws, instance_id, submission_id)
max_ram = _get_submission_max_ram(conf_aws, submission_id)
logger.info('Max ram usage of "{}": {}MB'.format(label, max_ram))
set_submission_max_ram(config, submission_id, max_ram)
logger.info('Downloading predictions of : "{}"'.format(label))
predictions_folder_path = download_predictions(
conf_aws, instance_id, submission_id)
set_predictions(config, submission_id, predictions_folder_path)
set_time(config, submission_id, predictions_folder_path)
set_scores(config, submission_id, predictions_folder_path)
set_submission_state(config, submission_id, 'tested')
logger.info('Scoring "{}"'.format(label))
score_submission(config, submission_id)
_run_hook(config, HOOK_SUCCESSFUL_TRAINING, submission_id)
else:
logger.info('Training of "{}" in "{}" failed'.format(
label, instance_id))
set_submission_state(config, submission_id, 'training_error')
error_msg = _get_traceback(
_get_log_content(conf_aws, submission_id))
set_submission_error_msg(config, submission_id, error_msg)
_run_hook(config, HOOK_FAILED_TRAINING, submission_id)
nb_trials += 1
time.sleep(conf_aws.get('new_instance_check_interval', 6))
_tag_instance_by_submission(conf_aws, instance.id, submission_name)
_add_or_update_tag(conf_aws, instance.id, 'train_loop', '1')
_add_or_update_tag(conf_aws, instance.id, 'event_name', event_name)
logger.info('Launched instance "{}" for submission "{}"'.format(
instance.id, submission))
set_submission_state(config, submission.id, 'sent_to_training')
# Score tested submissions
submissions = get_submissions(config, event_name, 'tested')
for submission_id, _ in submissions:
label = _get_submission_label_by_id(config, submission_id)
logger.info('Scoring submission : {}'.format(label))
score_submission(config, submission_id)
_run_hook(config, HOOK_SUCCESSFUL_TRAINING, submission_id)
# Get running instances and process events
instance_ids = list_ec2_instance_ids(conf_aws)
for instance_id in instance_ids:
if not _is_ready(conf_aws, instance_id):
continue
tags = _get_tags(conf_aws, instance_id)
# Filter instances that were not launched
# by the training loop API
# if 'submission_id' not in tags: # no longer added to tags
# continue
if tags.get('event_name') != event_name:
continue
if 'train_loop' not in tags:
continue
# Process each instance
submission_name = tags['Name']