Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
gradients_accum=gradients_accum,
train_steps=train_steps,
save_every=save_every,
report_every=report_every)
if __name__ == "__main__":
main(config.model_dir,
src_train_path=config.src_train_path,
tgt_train_path=config.tgt_train_path,
vocab_path=config.vocab_path,
maximum_length=config.maximum_length,
shuffle_buffer_size=config.shuffle_buffer_size,
gradients_accum=config.gradients_accum,
train_steps=config.train_steps,
save_every=config.save_every,
report_every=config.report_every)
# opennmt train model
train(model_dir,
inputter,
src_train_path,
tgt_train_path,
maximum_length=maximum_length,
shuffle_buffer_size=shuffle_buffer_size,
gradients_accum=gradients_accum,
train_steps=train_steps,
save_every=save_every,
report_every=report_every)
if __name__ == "__main__":
main(config.model_dir,
src_train_path=config.src_train_path,
tgt_train_path=config.tgt_train_path,
vocab_path=config.vocab_path,
maximum_length=config.maximum_length,
shuffle_buffer_size=config.shuffle_buffer_size,
gradients_accum=config.gradients_accum,
train_steps=config.train_steps,
save_every=config.save_every,
report_every=config.report_every)
_save_data(train_lst, train_src_path, train_tgt_path)
_save_data(test_lst, test_src_path, test_tgt_path)
def touch_empty_file(file_path):
with open(file_path, 'w', encoding='utf-8') as f:
f.write("")
if __name__ == '__main__':
# train data
data_list = []
for path in config.raw_train_paths:
data_list.extend(parse_xml_file(path))
transform_corpus_data(data_list,
config.src_train_path,
config.tgt_train_path,
config.src_test_path,
config.tgt_test_path)
maximum_length=maximum_length,
shuffle_buffer_size=shuffle_buffer_size,
gradients_accum=gradients_accum,
train_steps=train_steps,
save_every=save_every,
report_every=report_every)
if __name__ == "__main__":
main(config.model_dir,
src_train_path=config.src_train_path,
tgt_train_path=config.tgt_train_path,
vocab_path=config.vocab_path,
maximum_length=config.maximum_length,
shuffle_buffer_size=config.shuffle_buffer_size,
gradients_accum=config.gradients_accum,
train_steps=config.train_steps,
save_every=config.save_every,
report_every=config.report_every)
train_steps=train_steps,
save_every=save_every,
report_every=report_every)
if __name__ == "__main__":
main(config.model_dir,
src_train_path=config.src_train_path,
tgt_train_path=config.tgt_train_path,
vocab_path=config.vocab_path,
maximum_length=config.maximum_length,
shuffle_buffer_size=config.shuffle_buffer_size,
gradients_accum=config.gradients_accum,
train_steps=config.train_steps,
save_every=config.save_every,
report_every=config.report_every)
train(model_dir,
inputter,
src_train_path,
tgt_train_path,
maximum_length=maximum_length,
shuffle_buffer_size=shuffle_buffer_size,
gradients_accum=gradients_accum,
train_steps=train_steps,
save_every=save_every,
report_every=report_every)
if __name__ == "__main__":
main(config.model_dir,
src_train_path=config.src_train_path,
tgt_train_path=config.tgt_train_path,
vocab_path=config.vocab_path,
maximum_length=config.maximum_length,
shuffle_buffer_size=config.shuffle_buffer_size,
gradients_accum=config.gradients_accum,
train_steps=config.train_steps,
save_every=config.save_every,
report_every=config.report_every)
tgt_train_path,
maximum_length=maximum_length,
shuffle_buffer_size=shuffle_buffer_size,
gradients_accum=gradients_accum,
train_steps=train_steps,
save_every=save_every,
report_every=report_every)
if __name__ == "__main__":
main(config.model_dir,
src_train_path=config.src_train_path,
tgt_train_path=config.tgt_train_path,
vocab_path=config.vocab_path,
maximum_length=config.maximum_length,
shuffle_buffer_size=config.shuffle_buffer_size,
gradients_accum=config.gradients_accum,
train_steps=config.train_steps,
save_every=config.save_every,
report_every=config.report_every)
src_train_path,
tgt_train_path,
maximum_length=maximum_length,
shuffle_buffer_size=shuffle_buffer_size,
gradients_accum=gradients_accum,
train_steps=train_steps,
save_every=save_every,
report_every=report_every)
if __name__ == "__main__":
main(config.model_dir,
src_train_path=config.src_train_path,
tgt_train_path=config.tgt_train_path,
vocab_path=config.vocab_path,
maximum_length=config.maximum_length,
shuffle_buffer_size=config.shuffle_buffer_size,
gradients_accum=config.gradients_accum,
train_steps=config.train_steps,
save_every=config.save_every,
report_every=config.report_every)
"""
@author:XuMing(xuming624@qq.com)
@description:
"""
import sys
import tensorflow as tf
sys.path.append('../..')
from pycorrector.transformer import config
from pycorrector.transformer.model import translate, model, checkpoint
if __name__ == "__main__":
data_config = {
"source_vocabulary": config.src_vocab_path,
"target_vocabulary": config.tgt_vocab_path
}
model.initialize(data_config)
# Load model
checkpoint_manager = tf.train.CheckpointManager(checkpoint, config.model_dir, max_to_keep=5)
if checkpoint_manager.latest_checkpoint is not None:
tf.get_logger().info("Restoring parameters from %s", checkpoint_manager.latest_checkpoint)
checkpoint.restore(checkpoint_manager.latest_checkpoint)
translate(config.src_test_path,
batch_size=config.batch_size,
beam_size=config.beam_size)
})
# opennmt train model
train(model_dir,
inputter,
src_train_path,
tgt_train_path,
maximum_length=maximum_length,
shuffle_buffer_size=shuffle_buffer_size,
gradients_accum=gradients_accum,
train_steps=train_steps,
save_every=save_every,
report_every=report_every)
if __name__ == "__main__":
main(config.model_dir,
src_train_path=config.src_train_path,
tgt_train_path=config.tgt_train_path,
vocab_path=config.vocab_path,
maximum_length=config.maximum_length,
shuffle_buffer_size=config.shuffle_buffer_size,
gradients_accum=config.gradients_accum,
train_steps=config.train_steps,
save_every=config.save_every,
report_every=config.report_every)