Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
response = tokenize(example['labels'][0])
else:
response = tokenize(example['eval_labels'][0])
chosen_topic = tokenize(example['chosen_topic'])
# Set up knowledge
checked_knowledge = example['title'] + ' __knowledge__ ' + example['checked_sentence']
knowledges = [checked_knowledge] + \
[k for k in example['knowledge'].rstrip().split('\n')]
for idx, k in enumerate(knowledges[1:]):
if k == checked_knowledge:
break
else:
# Sometimes, knowledge does not include checked_sentnece
idx = None
colorlog.warning("Knowledge does not include checked sentence.")
if idx is not None:
del knowledges[idx + 1]
# Tokenize knowledge
knowledge_sentences = [tokenize(k) for k in knowledges]
new_example = {'context': context,
'response': response,
'chosen_topic': chosen_topic,
'knowledge_sentences': knowledge_sentences,
'episode_num': episode_num,
'example_num': example_num}
if 'multi_eval_labels' in example:
responses = [tokenize(response) for response in example['multi_eval_labels']]
new_example['responses'] = responses
if 'multi_checked_sentences' in example:
response = tokenize(example['labels'][0])
else:
response = tokenize(example['eval_labels'][0])
chosen_topic = tokenize(example['chosen_topic'])
# Set up knowledge
checked_knowledge = example['title'] + ' __knowledge__ ' + example['checked_sentence']
knowledges = [checked_knowledge] + \
[k for k in example['knowledge'].rstrip().split('\n')]
for idx, k in enumerate(knowledges[1:]):
if k == checked_knowledge:
break
else:
# Sometimes, knowledge does not include checked_sentnece
idx = None
colorlog.warning("Knowledge does not include checked sentence.")
if idx is not None:
del knowledges[idx + 1]
# Tokenize knowledge
knowledge_sentences = [tokenize(k) for k in knowledges]
new_example = {'context': context,
'response': response,
'chosen_topic': chosen_topic,
'knowledge_sentences': knowledge_sentences,
'episode_num': episode_num,
'example_num': example_num}
new_examples.append(new_example)
new_episodes.append(new_examples)
if self._datapath: