Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
"""Collect the results after that the submission is completed.
Be aware that calling ``collect_results()`` before that the submission
finished will lock the Python main process awaiting for the submission
to be processed. Use ``worker.status`` to know the status of the worker
beforehand.
"""
super().collect_results()
if self.status == 'finished' or self.status == 'running':
# communicate() will wait for the process to be completed
stdout, stderr = self._proc.communicate()
# combining stderr and stdout as errors might be piped to either
# (see https://github.com/paris-saclay-cds/ramp-board/issues/179)
# and extracting the error message from combined log
log_output = stdout + b'\n\n' + stderr
error_msg = _get_traceback(log_output.decode('utf-8'))
# write the log into the disk
log_dir = os.path.join(self.config['logs_dir'],
self.submission)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
with open(os.path.join(log_dir, 'log'), 'wb+') as f:
f.write(log_output)
# copy the predictions into the disk
# no need to create the directory, it will be handle by copytree
pred_dir = os.path.join(self.config['predictions_dir'],
self.submission)
output_training_dir = os.path.join(
self.config['submissions_dir'], self.submission,
'training_output')
if os.path.exists(pred_dir):
shutil.rmtree(pred_dir)
logger.info('Downloading predictions of : "{}"'.format(label))
predictions_folder_path = download_predictions(
conf_aws, instance_id, submission_id)
set_predictions(config, submission_id, predictions_folder_path)
set_time(config, submission_id, predictions_folder_path)
set_scores(config, submission_id, predictions_folder_path)
set_submission_state(config, submission_id, 'tested')
logger.info('Scoring "{}"'.format(label))
score_submission(config, submission_id)
_run_hook(config, HOOK_SUCCESSFUL_TRAINING, submission_id)
else:
logger.info('Training of "{}" in "{}" failed'.format(
label, instance_id))
set_submission_state(config, submission_id, 'training_error')
error_msg = _get_traceback(
_get_log_content(conf_aws, submission_id))
set_submission_error_msg(config, submission_id, error_msg)
_run_hook(config, HOOK_FAILED_TRAINING, submission_id)
config, submission_id, max_ram
)
logger.info('Downloading the predictions of "{}"'
.format(label))
path = download_predictions(
conf_aws, instance_id, submission_name)
set_predictions(config, submission_id, path)
set_time(config, submission_id, path)
set_scores(config, submission_id, path)
set_submission_state(config, submission_id, 'tested')
else:
logger.info('Training of "{}" failed'.format(label))
set_submission_state(
config, submission_id, 'training_error')
error_msg = _get_traceback(
_get_log_content(conf_aws, submission_name)
)
set_submission_error_msg(
config, submission_id, error_msg)
_run_hook(config, HOOK_FAILED_TRAINING, submission_id)
# training finished, so terminate the instance
terminate_ec2_instance(conf_aws, instance_id)
time.sleep(secs)
self.status = 'finished'
if self.status != 'finished':
raise ValueError("Cannot collect results if worker is not"
"'running' or 'finished'")
logger.info("Collecting submission '{}'".format(self.submission))
aws.download_log(self.config, self.instance.id, self.submission)
if aws._training_successful(
self.config, self.instance.id, self.submission):
_ = aws.download_predictions( # noqa
self.config, self.instance.id, self.submission)
self.status = 'collected'
exit_status, error_msg = 0, ''
else:
error_msg = _get_traceback(
aws._get_log_content(self.config, self.submission))
self.status = 'collected'
exit_status = 1
logger.info(repr(self))
return exit_status, error_msg