Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
if not audience:
audience = EXPECTED_OIDC_AUDIENCE
# checking that minimal audience and scope requirements (required by Rucio) are satisfied !
if not all_oidc_req_claims_present(auth_scope, audience, EXPECTED_OIDC_SCOPE, EXPECTED_OIDC_AUDIENCE):
raise CannotAuthenticate("Requirements of scope and audience do not satisfy minimal requirements of the Rucio server.")
issuer_id = kwargs.get('issuer', ADMIN_ISSUER_ID)
if not issuer_id:
issuer_id = ADMIN_ISSUER_ID
auto = kwargs.get('auto', False)
polling = kwargs.get('polling', False)
refresh_lifetime = kwargs.get('refresh_lifetime', REFRESH_LIFETIME_H)
ip = kwargs.get('ip', None)
webhome = kwargs.get('webhome', None)
# For webui a mock account will be used here and default account
# will be assigned to the identity during get_token_oidc
if account == InternalAccount('webui'):
pass
else:
# Make sure the account exists
if not account_exists(account, session=session):
return None
try:
start = time.time()
# redirect_url needs to be specified & one of those defined
# in the Rucio OIDC Client configuration
redirect_to = "auth/oidc_code"
if auto:
redirect_to = "auth/oidc_token"
# random strings in order to keep track of responses to outstanding requests (state)
# and to associate a client session with an ID Token and to mitigate replay attacks (nonce).
state, nonce = rndstr(50), rndstr(50)
if len(vo) != 3:
raise exception.RucioException('Invalid VO tag, must be 3 chars.')
new_vo = models.VO(vo=vo, description=description, email=email)
try:
new_vo.save(session=session)
except IntegrityError:
raise exception.Duplicate('VO {} already exists!'.format(vo))
except DatabaseError as error:
raise exception.RucioException(error.args)
from rucio.core.account import add_account, list_identities
from rucio.core.identity import add_account_identity
new_root = InternalAccount('root', vo=vo)
add_account(account=new_root, type=AccountType.from_sym('SERVICE'), email=email, session=session)
add_account_identity(identity='root@{}'.format(vo),
type=IdentityType.from_sym('userpass'),
account=new_root,
email=email,
default=False,
password='password',
session=session)
for ident in list_identities(account=InternalAccount('super_root', vo='def'), session=session):
add_account_identity(identity=ident['identity'], type=ident['type'], account=new_root, email='', session=session)
Update lock state of a replication rule.
:param rule_id: The rule_id to lock.
:param options: Options dictionary.
:param issuer: The issuing account of this operation
:param vo: The VO to act on.
:raises: RuleNotFound if no Rule can be found.
"""
kwargs = {'rule_id': rule_id, 'options': options}
if not has_permission(issuer=issuer, vo=vo, action='access_rule_vo', kwargs=kwargs):
raise AccessDenied('Account %s can not access rules at other VOs.' % (issuer))
if 'approve' in options:
if not has_permission(issuer=issuer, vo=vo, action='approve_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not approve/deny this replication rule.' % (issuer))
issuer = InternalAccount(issuer, vo=vo)
if options['approve']:
rule.approve_rule(rule_id=rule_id, approver=issuer)
else:
rule.deny_rule(rule_id=rule_id, approver=issuer, reason=options.get('comment', None))
else:
if not has_permission(issuer=issuer, vo=vo, action='update_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not update this replication rule.' % (issuer))
rule.update_rule(rule_id=rule_id, options=options)
replicas = []
rses = []
for report in self.__reports:
try:
# Identify suspicious files
try:
if self.__bad_files_patterns and report['eventType'] in ['get_sm', 'get_sm_a', 'get'] and 'clientState' in report and report['clientState'] not in ['DONE', 'FOUND_ROOT', 'ALREADY_DONE']:
for pattern in self.__bad_files_patterns:
if 'stateReason' in report and report['stateReason'] and isinstance(report['stateReason'], str) and pattern.match(report['stateReason']):
reason = report['stateReason'][:255]
if 'url' not in report or not report['url']:
logging.error('Missing url in the following trace : ' + str(report))
else:
try:
surl = report['url']
declare_bad_file_replicas([surl, ], reason=reason, issuer=InternalAccount('root'), status=BadFilesStatus.SUSPICIOUS)
logging.info('Declare suspicious file %s with reason %s' % (report['url'], reason))
except Exception as error:
logging.error('Failed to declare suspicious file' + str(error))
except Exception as error:
logging.error('Problem with bad trace : %s . Error %s' % (str(report), str(error)))
# check if scope in report. if not skip this one.
if 'scope' not in report:
record_counter('daemons.tracer.kronos.missing_scope')
if report['eventType'] != 'touch':
continue
else:
record_counter('daemons.tracer.kronos.with_scope')
report['scope'] = InternalScope(report['scope'])
# handle all events starting with get* and download and touch events.
if activity is None:
activity = 'User Subscriptions'
kwargs = {'dids': dids, 'copies': copies, 'rse_expression': rse_expression, 'weight': weight, 'lifetime': lifetime,
'grouping': grouping, 'account': account, 'locked': locked, 'subscription_id': subscription_id,
'source_replica_expression': source_replica_expression, 'notify': notify, 'activity': activity,
'purge_replicas': purge_replicas, 'ignore_availability': ignore_availability, 'comment': comment,
'ask_approval': ask_approval, 'asynchronous': asynchronous, 'priority': priority, 'split_container': split_container,
'meta': meta}
validate_schema(name='rule', obj=kwargs)
if not has_permission(issuer=issuer, vo=vo, action='add_rule', kwargs=kwargs):
raise AccessDenied('Account %s can not add replication rule' % (issuer))
account = InternalAccount(account, vo=vo)
for d in dids:
d['scope'] = InternalScope(d['scope'], vo=vo)
return rule.add_rule(account=account,
dids=dids,
copies=copies,
rse_expression=rse_expression,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=subscription_id,
source_replica_expression=source_replica_expression,
activity=activity,
notify=notify,
purge_replicas=purge_replicas,
def account_exists(account):
"""
Checks to see if account exists. This procedure does not check it's status.
:param account: Name of the account_core.
:returns: True if found, otherwise false.
"""
account = InternalAccount(account)
return account_core.account_exists(account)
def get_usage_history(rse, account, issuer):
"""
Returns historical values of the specified counter, or raises CounterNotFound if the counter does not exist.
:param rse: The RSE.
:param account: The account name.
:param issuer: The issuer account.
:returns: A dictionary with total and bytes.
"""
rse_id = get_rse_id(rse=rse)
account = InternalAccount(account)
return account_core.get_usage_history(rse_id, account)
if not isinstance(metadata, dict):
raise TypeError('metadata should be a dict')
if 'filter' in metadata and metadata['filter']:
if not isinstance(metadata['filter'], dict):
raise TypeError('filter should be a dict')
validate_schema(name='subscription_filter', obj=metadata['filter'])
if 'replication_rules' in metadata and metadata['replication_rules']:
if not isinstance(metadata['replication_rules'], list):
raise TypeError('replication_rules should be a list')
else:
for rule in metadata['replication_rules']:
validate_schema(name='activity', obj=rule.get('activity', 'default'))
except ValueError as error:
raise TypeError(error)
account = InternalAccount(account)
return subscription.update_subscription(name=name, account=account, metadata=metadata)
:returns: True is successful, False otherwise
"""
for v_file in files:
v_file.update({"type": "FILE"}) # Make sure DIDs are identified as files for checking
validate_schema(name='dids', obj=files)
rse_id = get_rse_id(rse=rse)
kwargs = {'rse': rse, 'rse_id': rse_id}
if not permission.has_permission(issuer=issuer, action='add_replicas', kwargs=kwargs):
raise exception.AccessDenied('Account %s can not add file replicas on %s' % (issuer, rse))
if not permission.has_permission(issuer=issuer, action='skip_availability_check', kwargs=kwargs):
ignore_availability = False
issuer = InternalAccount(issuer)
for f in files:
f['scope'] = InternalScope(f['scope'])
if 'account' in f:
f['account'] = InternalAccount(f['account'])
replica.add_replicas(rse_id=rse_id, files=files, account=issuer, ignore_availability=ignore_availability)
if surl_not_found:
cnt_surl_not_found += 1
logging.warning('replica_recoverer[%i/%i]: skipping suspicious replica %s on %s, no surls were found.', worker_number, total_workers, name, rse)
logging.info('replica_recoverer[%i/%i]: found %i/%i surls (took %.2f seconds), declaring them as bad replicas now.',
worker_number, total_workers, len(recoverable_replicas) - cnt_surl_not_found, len(recoverable_replicas), time.time() - start)
for vo in surls_to_recover:
for rse_id in surls_to_recover[vo]:
logging.info('replica_recoverer[%i/%i]: ready to declare %i bad replica(s) on %s: %s.',
worker_number, total_workers, len(surls_to_recover[vo][rse_id]), rse, str(surls_to_recover[vo][rse_id]))
if len(surls_to_recover[vo][rse_id]) > max_replicas_per_rse:
logging.warning('replica_recoverer[%i/%i]: encountered more than %i suspicious replicas (%s) on %s. Please investigate.',
worker_number, total_workers, max_replicas_per_rse, str(len(surls_to_recover[vo][rse_id])), rse)
else:
declare_bad_file_replicas(pfns=surls_to_recover[vo][rse_id], reason='Suspicious. Automatic recovery.', issuer=InternalAccount('root', vo=vo), status=BadFilesStatus.BAD, session=None)
logging.info('replica_recoverer[%i/%i]: finished declaring bad replicas on %s.', worker_number, total_workers, rse)
except (DatabaseException, DatabaseError) as err:
if match('.*QueuePool.*', str(err.args[0])):
logging.warning(traceback.format_exc())
record_counter('replica.recoverer.exceptions.%s', err.__class__.__name__)
elif match('.*ORA-03135.*', str(err.args[0])):
logging.warning(traceback.format_exc())
record_counter('replica.recoverer.exceptions.%s', err.__class__.__name__)
else:
logging.critical(traceback.format_exc())
record_counter('replica.recoverer.exceptions.%s', err.__class__.__name__)
except Exception as err:
logging.critical(traceback.format_exc())
record_counter('replica.recoverer.exceptions.%s', err.__class__.__name__)
if once: