github.com/pdaian/flashboys2@v0.0.0-20190718175736-b101c35361f0/write_csv.py (about) 1 import psycopg2, time, csv, os 2 3 def get_last_line(): 4 with open('arbitrage_data.csv', 'rb') as f: 5 f.seek(-2, os.SEEK_END) 6 while f.read(1) != b'\n': 7 f.seek(-2, os.SEEK_CUR) 8 return str(int(f.readline().decode().split(",")[-1])) # make sure it parses as int implicitly (or typeerror) 9 10 11 FIELDS_TO_GRAB = 'hash,monitor_ip,sender,time_seen,payload,gas_price,gas_limit,amount,peer_name,account_nonce,id' 12 13 conn = psycopg2.connect("postgres://readonly:NZOTyC4cttjvNdAKY@arbitrage3.ck0rrdngnqmh.us-west-2.rds.amazonaws.com/arbitrage?sslmode=verify-full") 14 cur = conn.cursor() 15 print(time.time()) 16 17 grab_from = get_last_line() 18 print("[database fetcher] Grabbing starting at id " + grab_from) 19 cur.execute("SELECT " + FIELDS_TO_GRAB + " FROM arbitrage WHERE id > " + grab_from + " AND id < " + str(int(grab_from) + 1000000) + ";") 20 print(time.time()) 21 22 with open('arbitrage_data.csv', 'a') as csvfile: 23 spamwriter = csv.writer(csvfile, delimiter=',', 24 quotechar='|', quoting=csv.QUOTE_MINIMAL) 25 26 #spamwriter.writerow(FIELDS_TO_GRAB.split(",")) 27 for item in cur.fetchall(): 28 spamwriter.writerow(item) 29 30 print("[database fetcher] Wrote to id " + get_last_line())