How to use the s3transfer.futures.TransferCoordinator function in s3transfer

To help you get started, we’ve selected a few s3transfer examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github boto / s3transfer / tests / unit / test_tasks.py View on Github external
pass


class ExceptionSubmissionTask(SubmissionTask):
    def _submit(self, transfer_future, executor=None, tasks_to_submit=None,
                additional_callbacks=None, exception=TaskFailureException):
        if executor and tasks_to_submit:
            for task_to_submit in tasks_to_submit:
                self._transfer_coordinator.submit(executor, task_to_submit)
        if additional_callbacks:
            for callback in additional_callbacks:
                callback()
        raise exception()


class StatusRecordingTransferCoordinator(TransferCoordinator):
    def __init__(self, transfer_id=None):
        super(StatusRecordingTransferCoordinator, self).__init__(transfer_id)
        self.status_changes = [self._status]

    def set_status_to_queued(self):
        super(StatusRecordingTransferCoordinator, self).set_status_to_queued()
        self._record_status_change()

    def set_status_to_running(self):
        super(StatusRecordingTransferCoordinator, self).set_status_to_running()
        self._record_status_change()

    def _record_status_change(self):
        self.status_changes.append(self._status)
github boto / s3transfer / tests / unit / test_futures.py View on Github external
def setUp(self):
        self.transfer_coordinator = TransferCoordinator()
github boto / s3transfer / tests / unit / test_futures.py View on Github external
def return_call_args(*args, **kwargs):
    return args, kwargs


def raise_exception(exception):
    raise exception


def get_exc_info(exception):
    try:
        raise_exception(exception)
    except:
        return sys.exc_info()


class RecordingTransferCoordinator(TransferCoordinator):
    def __init__(self):
        self.all_transfer_futures_ever_associated = set()
        super(RecordingTransferCoordinator, self).__init__()

    def add_associated_future(self, future):
        self.all_transfer_futures_ever_associated.add(future)
        super(RecordingTransferCoordinator, self).add_associated_future(future)


class ReturnFooTask(Task):
    def _main(self, **kwargs):
        return 'foo'


class SleepTask(Task):
    def _main(self, sleep_time, **kwargs):
github boto / s3transfer / tests / __init__.py View on Github external
def setUp(self):
        super(BaseTaskTest, self).setUp()
        self.transfer_coordinator = TransferCoordinator()
github boto / s3transfer / tests / unit / test_futures.py View on Github external
def test_transfer_id(self):
        transfer_coordinator = TransferCoordinator(transfer_id=1)
        self.assertEqual(transfer_coordinator.transfer_id, 1)
github boto / s3transfer / tests / unit / test_futures.py View on Github external
def test_repr(self):
        transfer_coordinator = TransferCoordinator(transfer_id=1)
        self.assertEqual(
            repr(transfer_coordinator), 'TransferCoordinator(transfer_id=1)')
github boto / s3transfer / tests / unit / test_bandwidth.py View on Github external
def setUp(self):
        self.leaky_bucket = mock.Mock(LeakyBucket)
        self.time_utils = mock.Mock(TimeUtils)
        self.tempdir = tempfile.mkdtemp()
        self.content = b'a' * 1024 * 1024
        self.filename = os.path.join(self.tempdir, 'myfile')
        with open(self.filename, 'wb') as f:
            f.write(self.content)
        self.coordinator = TransferCoordinator()
github trulia / cidr-house-rules / vendor / s3transfer / manager.py View on Github external
def _get_future_with_components(self, call_args):
        transfer_id = self._id_counter
        # Creates a new transfer future along with its components
        transfer_coordinator = TransferCoordinator(transfer_id=transfer_id)
        # Track the transfer coordinator for transfers to manage.
        self._coordinator_controller.add_transfer_coordinator(
            transfer_coordinator)
        # Also make sure that the transfer coordinator is removed once
        # the transfer completes so it does not stick around in memory.
        transfer_coordinator.add_done_callback(
            self._coordinator_controller.remove_transfer_coordinator,
            transfer_coordinator)
        components = {
            'meta': TransferMeta(call_args, transfer_id=transfer_id),
            'coordinator': transfer_coordinator
        }
        transfer_future = TransferFuture(**components)
        return transfer_future, components
github boto / s3transfer / s3transfer / futures.py View on Github external
:type meta: TransferMeta
        :param meta: The metadata associated to the request. This object
            is visible to the requester.

        :type coordinator: TransferCoordinator
        :param coordinator: The coordinator associated to the request. This
            object is not visible to the requester.
        """
        self._meta = meta
        if meta is None:
            self._meta = TransferMeta()

        self._coordinator = coordinator
        if coordinator is None:
            self._coordinator = TransferCoordinator()