github.com/pachyderm/pachyderm@v1.13.4/examples/ml/gpt-2/train.py (about)

     1  #!/usr/bin/python3
     2  import gpt_2_simple as gpt2
     3  import os
     4  
     5  
     6  tweets = [f for f in os.listdir("/pfs/tweets")]
     7  
     8  # chdir so that the training process outputs to the right place
     9  out = os.path.join("/pfs/out", tweets[0])
    10  os.mkdir(out)
    11  # chdir to get gpt2 to output where want it to
    12  os.chdir(out)
    13  
    14  model_name = "345M"
    15  gpt2.download_gpt2(model_name=model_name)
    16  
    17  
    18  sess = gpt2.start_tf_sess()
    19  gpt2.finetune(sess,
    20                os.path.join("/pfs/tweets", tweets[0]),
    21                model_name=model_name,
    22                steps=1000)   # steps is max number of training steps