How to use the ramp-engine.ramp_engine.base._get_traceback function in ramp-engine

To help you get started, we’ve selected a few ramp-engine examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github paris-saclay-cds / ramp-board / ramp-engine / ramp_engine / local.py View on Github external
"""Collect the results after that the submission is completed.

        Be aware that calling ``collect_results()`` before that the submission
        finished will lock the Python main process awaiting for the submission
        to be processed. Use ``worker.status`` to know the status of the worker
        beforehand.
        """
        super().collect_results()
        if self.status == 'finished' or self.status == 'running':
            # communicate() will wait for the process to be completed
            stdout, stderr = self._proc.communicate()
            # combining stderr and stdout as errors might be piped to either
            # (see https://github.com/paris-saclay-cds/ramp-board/issues/179)
            # and extracting the error message from combined log
            log_output = stdout + b'\n\n' + stderr
            error_msg = _get_traceback(log_output.decode('utf-8'))
            # write the log into the disk
            log_dir = os.path.join(self.config['logs_dir'],
                                   self.submission)
            if not os.path.exists(log_dir):
                os.makedirs(log_dir)
            with open(os.path.join(log_dir, 'log'), 'wb+') as f:
                f.write(log_output)
            # copy the predictions into the disk
            # no need to create the directory, it will be handle by copytree
            pred_dir = os.path.join(self.config['predictions_dir'],
                                    self.submission)
            output_training_dir = os.path.join(
                self.config['submissions_dir'], self.submission,
                'training_output')
            if os.path.exists(pred_dir):
                shutil.rmtree(pred_dir)
github paris-saclay-cds / ramp-board / ramp-engine / ramp_engine / aws / aws_train.py View on Github external
logger.info('Downloading predictions of : "{}"'.format(label))
        predictions_folder_path = download_predictions(
            conf_aws, instance_id, submission_id)
        set_predictions(config, submission_id, predictions_folder_path)
        set_time(config, submission_id, predictions_folder_path)
        set_scores(config, submission_id, predictions_folder_path)
        set_submission_state(config, submission_id, 'tested')
        logger.info('Scoring "{}"'.format(label))
        score_submission(config, submission_id)
        _run_hook(config, HOOK_SUCCESSFUL_TRAINING, submission_id)
    else:
        logger.info('Training of "{}" in "{}" failed'.format(
            label, instance_id))
        set_submission_state(config, submission_id, 'training_error')
        error_msg = _get_traceback(
            _get_log_content(conf_aws, submission_id))
        set_submission_error_msg(config, submission_id, error_msg)
        _run_hook(config, HOOK_FAILED_TRAINING, submission_id)
github paris-saclay-cds / ramp-board / ramp-engine / ramp_engine / aws / aws_train.py View on Github external
config, submission_id, max_ram
                            )

                        logger.info('Downloading the predictions of "{}"'
                                    .format(label))
                        path = download_predictions(
                            conf_aws, instance_id, submission_name)
                        set_predictions(config, submission_id, path)
                        set_time(config, submission_id, path)
                        set_scores(config, submission_id, path)
                        set_submission_state(config, submission_id, 'tested')
                    else:
                        logger.info('Training of "{}" failed'.format(label))
                        set_submission_state(
                            config, submission_id, 'training_error')
                        error_msg = _get_traceback(
                            _get_log_content(conf_aws, submission_name)
                        )
                        set_submission_error_msg(
                            config, submission_id, error_msg)
                        _run_hook(config, HOOK_FAILED_TRAINING, submission_id)
                    # training finished, so terminate the instance
                    terminate_ec2_instance(conf_aws, instance_id)
        time.sleep(secs)
github paris-saclay-cds / ramp-board / ramp-engine / ramp_engine / aws / worker.py View on Github external
self.status = 'finished'
        if self.status != 'finished':
            raise ValueError("Cannot collect results if worker is not"
                             "'running' or 'finished'")

        logger.info("Collecting submission '{}'".format(self.submission))
        aws.download_log(self.config, self.instance.id, self.submission)

        if aws._training_successful(
                self.config, self.instance.id, self.submission):
            _ = aws.download_predictions(  # noqa
                self.config, self.instance.id, self.submission)
            self.status = 'collected'
            exit_status, error_msg = 0, ''
        else:
            error_msg = _get_traceback(
                aws._get_log_content(self.config, self.submission))
            self.status = 'collected'
            exit_status = 1
        logger.info(repr(self))
        return exit_status, error_msg