github.com/pdaian/flashboys2@v0.0.0-20190718175736-b101c35361f0/write_csv.py (about)

     1  import psycopg2, time, csv, os
     2  
     3  def get_last_line():
     4      with open('arbitrage_data.csv', 'rb') as f:
     5          f.seek(-2, os.SEEK_END)
     6          while f.read(1) != b'\n':
     7              f.seek(-2, os.SEEK_CUR) 
     8          return str(int(f.readline().decode().split(",")[-1])) # make sure it parses as int implicitly (or typeerror)
     9  
    10  
    11  FIELDS_TO_GRAB = 'hash,monitor_ip,sender,time_seen,payload,gas_price,gas_limit,amount,peer_name,account_nonce,id'
    12  
    13  conn = psycopg2.connect("postgres://readonly:NZOTyC4cttjvNdAKY@arbitrage3.ck0rrdngnqmh.us-west-2.rds.amazonaws.com/arbitrage?sslmode=verify-full")
    14  cur = conn.cursor()
    15  print(time.time())
    16  
    17  grab_from = get_last_line()
    18  print("[database fetcher] Grabbing starting at id " + grab_from)
    19  cur.execute("SELECT " + FIELDS_TO_GRAB + " FROM arbitrage WHERE id > " + grab_from +  " AND id < " + str(int(grab_from) + 1000000) + ";")
    20  print(time.time())
    21  
    22  with open('arbitrage_data.csv', 'a') as csvfile:
    23      spamwriter = csv.writer(csvfile, delimiter=',',
    24                              quotechar='|', quoting=csv.QUOTE_MINIMAL)
    25  
    26      #spamwriter.writerow(FIELDS_TO_GRAB.split(","))
    27      for item in cur.fetchall():
    28          spamwriter.writerow(item)
    29  
    30  print("[database fetcher] Wrote to id " + get_last_line())