Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Resolve the new_child_dids to its locks
with record_timer_block('rule.evaluate_did_attach.resolve_did_to_locks_and_replicas'):
# Resolve the rules to possible target rses:
possible_rses = []
source_rses = []
if session.bind.dialect.name != 'sqlite':
session.begin_nested()
try:
for rule in rules:
if rule.source_replica_expression:
source_rses.extend(parse_expression(rule.source_replica_expression, session=session))
if rule.ignore_availability:
possible_rses.extend(parse_expression(rule.rse_expression, session=session))
else:
possible_rses.extend(parse_expression(rule.rse_expression, filter={'availability_write': True}, session=session))
except Exception, e:
logging.warning('Could not parse RSE expression for possible RSEs for rule %s' % (str(rule.id)))
session.rollback()
possible_rses = []
session.begin_nested()
if session.bind.dialect.name != 'sqlite':
session.commit()
source_rses = list(set([rse['id'] for rse in source_rses]))
possible_rses = list(set([rse['id'] for rse in possible_rses]))
datasetfiles, locks, replicas, source_replicas = __resolve_dids_to_locks_and_replicas(dids=new_child_dids,
nowait=True,
restrict_rses=possible_rses,
source_rses=source_rses,
session=session)
:param kwargs: List of arguments for the action.
:returns: True if account is allowed to call the API call, otherwise False
"""
if _is_root(issuer):
return True
if get_rule(kwargs['rule_id'])['account'] == issuer:
return True
# Check if user is a country admin
admin_in_country = []
for kv in list_account_attributes(account=issuer):
if kv['key'].startswith('country-') and kv['value'] == 'admin':
admin_in_country.append(kv['key'].partition('-')[2])
rule = get_rule(rule_id=kwargs['rule_id'])
rses = parse_expression(rule['rse_expression'])
if admin_in_country:
for rse in rses:
if list_rse_attributes(rse_id=rse['id']).get('country') in admin_in_country:
return True
# DELETERS can approve the rule
for rse in rses:
rse_attr = list_rse_attributes(rse_id=rse['id'])
if rse_attr.get('rule_deleters'):
if issuer.external in rse_attr.get('rule_deleters').split(','):
return True
return False
if ignore_availability:
rses = parse_expression(rse_expression, session=session)
else:
rses = parse_expression(rse_expression, filter={'availability_write': True}, session=session)
if lifetime is None: # Check if one of the rses is a staging area
if [rse for rse in rses if rse.get('staging_area', False)]:
raise StagingAreaRuleRequiresLifetime()
# Auto-lock rules for TAPE rses
if not locked:
if [rse for rse in rses if rse.get('rse_type', RSEType.DISK) == RSEType.TAPE]:
locked = True
if source_replica_expression:
source_rses = parse_expression(source_replica_expression, session=session)
else:
source_rses = []
# 2. Create the rse selector
with record_timer_block('rule.add_rule.create_rse_selector'):
rseselector = RSESelector(account=account, rses=rses, weight=weight, copies=copies, ignore_account_limit=ask_approval, session=session)
expires_at = datetime.utcnow() + timedelta(seconds=lifetime) if lifetime is not None else None
if notify == 'Y':
notify = RuleNotification.YES
elif notify == 'C':
notify = RuleNotification.CLOSE
else:
notify = RuleNotification.NO
if rses:
invalid = set(rses) - set([rse['rse'] for rse in all_rses])
if invalid:
msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
', '.join([repr(rse) for rse in invalid]))
raise RSENotFound(msg)
rses = [rse for rse in all_rses if rse['rse'] in rses]
else:
rses = all_rses
if exclude_rses:
excluded_rses = parse_expression(exclude_rses)
rses = [rse for rse in rses if rse not in excluded_rses]
if include_rses:
included_rses = parse_expression(include_rses)
rses = [rse for rse in rses if rse in included_rses]
if not rses:
logging.error('Reaper: No RSEs found. Exiting.')
return
logging.info('Reaper: This instance will work on RSEs: ' + ', '.join([rse['rse'] for rse in rses]))
threads = []
nb_rses_per_worker = int(math.ceil(len(rses) / float(total_workers))) or 1
rses = random.sample(rses, len(rses))
for worker in range(total_workers):
for child in range(threads_per_worker or 1):
rses_list = rses[worker * nb_rses_per_worker: worker * nb_rses_per_worker + nb_rses_per_worker]
if not rses_list:
logging.warning('Reaper: Empty RSEs list for worker %(worker)s' % locals())
"""
Checks if an account can delete a global account limit.
:param issuer: Account identifier which issues the command.
:param kwargs: List of arguments for the action.
:returns: True if account is allowed, otherwise False
"""
if _is_root(issuer) or has_account_attribute(account=issuer, key='admin'):
return True
# Check if user is a country admin
admin_in_country = set()
for kv in list_account_attributes(account=issuer):
if kv['key'].startswith('country-') and kv['value'] == 'admin':
admin_in_country.add(kv['key'].partition('-')[2])
if admin_in_country:
resolved_rse_countries = {list_rse_attributes(rse_id=rse['rse_id']).get('country') for rse in parse_expression(kwargs['rse_exp'])}
if resolved_rse_countries.issubset(admin_in_country):
return True
return False
attemptnr = 0
skip_rule_creation = False
selected_rses = []
chained_idx = rule_dict.get('chained_idx', None)
if chained_idx:
params = {}
if rule_dict.get('associated_site_idx', None):
params['associated_site_idx'] = rule_dict.get('associated_site_idx', None)
logging.debug('%s Chained subscription identified. Will use %s', prepend_str, str(created_rules[chained_idx]))
algorithm = rule_dict.get('algorithm', None)
selected_rses = select_algorithm(algorithm, created_rules[chained_idx], params)
else:
# In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm
if split_rule:
rses = parse_expression(rse_expression)
list_of_rses = [rse['id'] for rse in rses]
# Check that some rule doesn't already exist for this DID and subscription
preferred_rse_ids = []
for rule in list_rules(filters={'subscription_id': subscription_id, 'scope': did['scope'], 'name': did['name']}):
already_existing_rses = [(rse['rse'], rse['id']) for rse in parse_expression(rule['rse_expression'])]
for rse, rse_id in already_existing_rses:
if (rse_id in list_of_rses) and (rse_id not in preferred_rse_ids):
preferred_rse_ids.append(rse_id)
if len(preferred_rse_ids) >= copies:
skip_rule_creation = True
rse_id_dict = {}
for rse in rses:
rse_id_dict[rse['id']] = rse['rse']
try:
rseselector = RSESelector(account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids))
selected_rses = [rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse(0, preferred_rse_ids=preferred_rse_ids, copies=copies, blacklist=blacklisted_rse_id)]
def site_selector(replicas, site):
"""
Return a list of replicas located on one site.
:param replicas : A dict with RSEs as values and replicas as keys (URIs).
:param site : The name of the site
"""
result = []
try:
rses = parse_expression("site=%s" % site)
except InvalidRSEExpression:
return result
except Exception:
return result
rses = [i['rse'] for i in rses]
for replica in replicas:
if replicas[replica] in rses:
result.append(replica)
return result
:param sign_urls: If set, will sign the PFNs if necessary.
:param signature_lifetime: If supported, in seconds, restrict the lifetime of the signed PFN.
:param resolve_archives: When set to true, find archives which contain the replicas.
:param resolve_parents: When set to true, find all parent datasets which contain the replicas.
:param session: The database session in use.
"""
file_clause, dataset_clause, state_clause, files, constituents = _resolve_dids(dids=dids, unavailable=unavailable,
ignore_availability=ignore_availability,
all_states=all_states,
resolve_archives=resolve_archives,
session=session)
rse_clause = []
if rse_expression:
for rse in parse_expression(expression=rse_expression, session=session):
rse_clause.append(models.RSEFileAssociation.rse_id == rse['id'])
for f in _list_replicas(dataset_clause, file_clause, state_clause, pfns,
schemes, files, rse_clause, rse_expression, client_location, domain,
sign_urls, signature_lifetime, constituents, resolve_parents, session):
yield f
"""
Get a list of rses for conveyor
:param rses: List of rses
:param include_rses: RSEs to include
:param exclude_rses: RSEs to exclude
:return: List of working rses
"""
working_rses = []
rses_list = list_rses()
if rses:
working_rses = [rse for rse in rses_list if rse['rse'] in rses]
if include_rses:
try:
parsed_rses = parse_expression(include_rses, session=None)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s to include RSEs", include_rses)
else:
for rse in parsed_rses:
if rse not in working_rses:
working_rses.append(rse)
if not (rses or include_rses):
working_rses = rses_list
if exclude_rses:
try:
parsed_rses = parse_expression(exclude_rses, session=None)
except InvalidRSEExpression as error:
logging.error("Invalid RSE exception %s to exclude RSEs: %s", exclude_rses, error)
else: