How to use the pycorrector.transformer.config function in pycorrector

To help you get started, we’ve selected a few pycorrector examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github shibing624 / pycorrector / pycorrector / transformer / train.py View on Github external
gradients_accum=gradients_accum,
          train_steps=train_steps,
          save_every=save_every,
          report_every=report_every)


if __name__ == "__main__":
    main(config.model_dir,
         src_train_path=config.src_train_path,
         tgt_train_path=config.tgt_train_path,
         vocab_path=config.vocab_path,
         maximum_length=config.maximum_length,
         shuffle_buffer_size=config.shuffle_buffer_size,
         gradients_accum=config.gradients_accum,
         train_steps=config.train_steps,
         save_every=config.save_every,
         report_every=config.report_every)
github shibing624 / pycorrector / pycorrector / transformer / train.py View on Github external
# opennmt train model
    train(model_dir,
          inputter,
          src_train_path,
          tgt_train_path,
          maximum_length=maximum_length,
          shuffle_buffer_size=shuffle_buffer_size,
          gradients_accum=gradients_accum,
          train_steps=train_steps,
          save_every=save_every,
          report_every=report_every)


if __name__ == "__main__":
    main(config.model_dir,
         src_train_path=config.src_train_path,
         tgt_train_path=config.tgt_train_path,
         vocab_path=config.vocab_path,
         maximum_length=config.maximum_length,
         shuffle_buffer_size=config.shuffle_buffer_size,
         gradients_accum=config.gradients_accum,
         train_steps=config.train_steps,
         save_every=config.save_every,
         report_every=config.report_every)
github shibing624 / pycorrector / pycorrector / transformer / preprocess.py View on Github external
_save_data(train_lst, train_src_path, train_tgt_path)
    _save_data(test_lst, test_src_path, test_tgt_path)


def touch_empty_file(file_path):
    with open(file_path, 'w', encoding='utf-8') as f:
        f.write("")


if __name__ == '__main__':
    # train data
    data_list = []
    for path in config.raw_train_paths:
        data_list.extend(parse_xml_file(path))
    transform_corpus_data(data_list,
                          config.src_train_path,
                          config.tgt_train_path,
                          config.src_test_path,
                          config.tgt_test_path)
github shibing624 / pycorrector / pycorrector / transformer / train.py View on Github external
maximum_length=maximum_length,
          shuffle_buffer_size=shuffle_buffer_size,
          gradients_accum=gradients_accum,
          train_steps=train_steps,
          save_every=save_every,
          report_every=report_every)


if __name__ == "__main__":
    main(config.model_dir,
         src_train_path=config.src_train_path,
         tgt_train_path=config.tgt_train_path,
         vocab_path=config.vocab_path,
         maximum_length=config.maximum_length,
         shuffle_buffer_size=config.shuffle_buffer_size,
         gradients_accum=config.gradients_accum,
         train_steps=config.train_steps,
         save_every=config.save_every,
         report_every=config.report_every)
github shibing624 / pycorrector / pycorrector / transformer / train.py View on Github external
train_steps=train_steps,
          save_every=save_every,
          report_every=report_every)


if __name__ == "__main__":
    main(config.model_dir,
         src_train_path=config.src_train_path,
         tgt_train_path=config.tgt_train_path,
         vocab_path=config.vocab_path,
         maximum_length=config.maximum_length,
         shuffle_buffer_size=config.shuffle_buffer_size,
         gradients_accum=config.gradients_accum,
         train_steps=config.train_steps,
         save_every=config.save_every,
         report_every=config.report_every)
github shibing624 / pycorrector / pycorrector / transformer / train.py View on Github external
train(model_dir,
          inputter,
          src_train_path,
          tgt_train_path,
          maximum_length=maximum_length,
          shuffle_buffer_size=shuffle_buffer_size,
          gradients_accum=gradients_accum,
          train_steps=train_steps,
          save_every=save_every,
          report_every=report_every)


if __name__ == "__main__":
    main(config.model_dir,
         src_train_path=config.src_train_path,
         tgt_train_path=config.tgt_train_path,
         vocab_path=config.vocab_path,
         maximum_length=config.maximum_length,
         shuffle_buffer_size=config.shuffle_buffer_size,
         gradients_accum=config.gradients_accum,
         train_steps=config.train_steps,
         save_every=config.save_every,
         report_every=config.report_every)
github shibing624 / pycorrector / pycorrector / transformer / train.py View on Github external
tgt_train_path,
          maximum_length=maximum_length,
          shuffle_buffer_size=shuffle_buffer_size,
          gradients_accum=gradients_accum,
          train_steps=train_steps,
          save_every=save_every,
          report_every=report_every)


if __name__ == "__main__":
    main(config.model_dir,
         src_train_path=config.src_train_path,
         tgt_train_path=config.tgt_train_path,
         vocab_path=config.vocab_path,
         maximum_length=config.maximum_length,
         shuffle_buffer_size=config.shuffle_buffer_size,
         gradients_accum=config.gradients_accum,
         train_steps=config.train_steps,
         save_every=config.save_every,
         report_every=config.report_every)
github shibing624 / pycorrector / pycorrector / transformer / train.py View on Github external
src_train_path,
          tgt_train_path,
          maximum_length=maximum_length,
          shuffle_buffer_size=shuffle_buffer_size,
          gradients_accum=gradients_accum,
          train_steps=train_steps,
          save_every=save_every,
          report_every=report_every)


if __name__ == "__main__":
    main(config.model_dir,
         src_train_path=config.src_train_path,
         tgt_train_path=config.tgt_train_path,
         vocab_path=config.vocab_path,
         maximum_length=config.maximum_length,
         shuffle_buffer_size=config.shuffle_buffer_size,
         gradients_accum=config.gradients_accum,
         train_steps=config.train_steps,
         save_every=config.save_every,
         report_every=config.report_every)
github shibing624 / pycorrector / pycorrector / transformer / infer.py View on Github external
"""
@author:XuMing(xuming624@qq.com)
@description: 
"""
import sys

import tensorflow as tf

sys.path.append('../..')

from pycorrector.transformer import config
from pycorrector.transformer.model import translate, model, checkpoint

if __name__ == "__main__":
    data_config = {
        "source_vocabulary": config.src_vocab_path,
        "target_vocabulary": config.tgt_vocab_path
    }

    model.initialize(data_config)
    # Load model
    checkpoint_manager = tf.train.CheckpointManager(checkpoint, config.model_dir, max_to_keep=5)
    if checkpoint_manager.latest_checkpoint is not None:
        tf.get_logger().info("Restoring parameters from %s", checkpoint_manager.latest_checkpoint)
        checkpoint.restore(checkpoint_manager.latest_checkpoint)

    translate(config.src_test_path,
              batch_size=config.batch_size,
              beam_size=config.beam_size)
github shibing624 / pycorrector / pycorrector / transformer / train.py View on Github external
})
    # opennmt train model
    train(model_dir,
          inputter,
          src_train_path,
          tgt_train_path,
          maximum_length=maximum_length,
          shuffle_buffer_size=shuffle_buffer_size,
          gradients_accum=gradients_accum,
          train_steps=train_steps,
          save_every=save_every,
          report_every=report_every)


if __name__ == "__main__":
    main(config.model_dir,
         src_train_path=config.src_train_path,
         tgt_train_path=config.tgt_train_path,
         vocab_path=config.vocab_path,
         maximum_length=config.maximum_length,
         shuffle_buffer_size=config.shuffle_buffer_size,
         gradients_accum=config.gradients_accum,
         train_steps=config.train_steps,
         save_every=config.save_every,
         report_every=config.report_every)