github.com/zartbot/goflow@v0.0.0-20221001080520-456f7dfad288/tools/parse_ipfix.py (about) 1 import pandas as pd 2 import re 3 cisco = pd.read_excel('./nf.xlsx',sheet_name="Sheet2") 4 cisco = cisco[(cisco['Val'].notnull()) & (cisco['Field Type'].notnull())] 5 regexp = re.compile(r'\(.*\)') 6 cisco['FullID']=cisco['Val'].replace(regexp,'').astype(int) 7 cisco = cisco[cisco['FullID']<65000] 8 cisco['ElementID'] = cisco['FullID'] - 32768 9 cisco['EnterpriseNo'] = 9 10 cisco['FieldType'] = cisco['Field Type'].apply(lambda x: x.replace('+','').replace(' ','_').replace('(','').replace(')','_').replace('%','')) 11 cisco = cisco[['FullID','ElementID','EnterpriseNo','FieldType','Len','Description']].reset_index(drop=True) 12 cisco['Description'].fillna('',inplace=True) 13 cisco['Len'] = cisco['Len'].apply(lambda x: str(x).replace('\n','none')) 14 #cisco['DataType'] = 'octetArray' 15 cisco.loc[cisco['FieldType'].str.contains('IPv4Address',regex=False),'DataType'] = 'ipv4Address' 16 cisco.loc[cisco['FieldType'].str.contains('IPv6Address',regex=False),'DataType'] = 'ipv6Address' 17 cisco.loc[cisco['FieldType'].str.contains('AddressIPv4',regex=False),'DataType'] = 'ipv4Address' 18 cisco.loc[cisco['FieldType'].str.contains('AddressIPv6',regex=False),'DataType'] = 'ipv6Address' 19 cisco.loc[(cisco['Description'].str.contains('addr',case=False) & cisco['Description'].str.contains('v4',case=False)) ,'DataType'] = 'ipv4Address' 20 cisco.loc[(cisco['Description'].str.contains('addr',case=False) & cisco['Description'].str.contains('v6',case=False)) ,'DataType'] = 'ipv6Address' 21 22 cisco.loc[cisco['FieldType'].str.contains('uri',case=False,regex=False),'DataType'] = 'string' 23 cisco.loc[cisco['FieldType'].str.contains('string',case=False,regex=False),'DataType'] = 'string' 24 cisco.loc[cisco['Description'].str.contains('string',case=False),'DataType'] = 'string' 25 cisco.loc[cisco['Len'].str.contains('string',case=False),'DataType'] = 'string' 26 cisco.loc[cisco['Len'].str.contains('Bitstring',case=False),'DataType'] = 'octetArray' 27 cisco.loc[cisco['Len'].str.contains('u8',case=False),'DataType'] = 'unsigned8' 28 cisco.loc[cisco['Len'].str.contains('u16',case=False),'DataType'] = 'unsigned16' 29 cisco.loc[cisco['Len'].str.contains('u32',case=False),'DataType'] = 'unsigned32' 30 cisco.loc[cisco['Len'].str.contains('u64',case=False),'DataType'] = 'unsigned64' 31 cisco.loc[cisco['FieldType'].str.contains('Port'),'DataType'] ='unsigned16' 32 cisco.loc[cisco['FieldType'].str.contains('Port',case=False) & (cisco['Len'] == '2'),'DataType'] = 'unsigned16' 33 cisco.loc[ (cisco['FieldType'].str.contains('count',case=False) & (cisco['Len'] == '1')),'DataType'] = 'unsigned8' 34 cisco.loc[ (cisco['FieldType'].str.contains('count',case=False) & (cisco['Len'] == '2')),'DataType'] = 'unsigned16' 35 cisco.loc[ (cisco['FieldType'].str.contains('count',case=False) & (cisco['Len'] == '4')),'DataType'] = 'unsigned32' 36 cisco.loc[ (cisco['FieldType'].str.contains('count',case=False) & (cisco['Len'] == '8')),'DataType'] = 'unsigned64' 37 cisco.loc[(cisco['Description'].str.contains('number',case=False) & (cisco['Len'] == '1')),'DataType'] = 'unsigned8' 38 cisco.loc[(cisco['Description'].str.contains('number',case=False) & (cisco['Len'] == '2')),'DataType'] = 'unsigned16' 39 cisco.loc[(cisco['Description'].str.contains('number',case=False) & (cisco['Len'] == '4')),'DataType'] = 'unsigned32' 40 cisco.loc[(cisco['Description'].str.contains('number',case=False) & (cisco['Len'] == '8')),'DataType'] = 'unsigned64' 41 cisco.loc[(cisco['Description'].str.contains('number',case=False) & (cisco['Len'] == '4 or 8')),'DataType'] = 'unsigned64' 42 cisco.loc[(cisco['Description'].str.contains('msec',case=False)),'DataType'] = 'dateTimeMilliseconds' 43 cisco.loc[(cisco['Description'].str.contains('id',case=False)& (cisco['Len'] == '1')),'DataType'] = 'unsigned8' 44 cisco.loc[(cisco['Description'].str.contains('id',case=False)& (cisco['Len'] == '2')),'DataType'] = 'unsigned16' 45 cisco.loc[(cisco['Description'].str.contains('id',case=False)& (cisco['Len'] == '4')),'DataType'] = 'unsigned32' 46 cisco.loc[(cisco['Description'].str.contains('id',case=False)& (cisco['Len'] == '8')),'DataType'] = 'unsigned64' 47 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '1')),'DataType'] = 'unsigned8' 48 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '2')),'DataType'] = 'unsigned16' 49 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '4')),'DataType'] = 'unsigned32' 50 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '8')),'DataType'] = 'unsigned64' 51 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '32bits')),'DataType'] = 'unsigned32' 52 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '64bits')),'DataType'] = 'unsigned64' 53 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == 'IPV4')),'DataType'] = 'ipv4Address' 54 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == 'IPv4')),'DataType'] = 'ipv4Address' 55 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == 'IPV6')),'DataType'] = 'ipv6Address' 56 cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == 'IPv6')),'DataType'] = 'ipv6Address' 57 cisco.loc[(cisco['DataType'].isna()),'DataType'] = 'octetArray' 58 cisco = cisco[['FullID','ElementID','EnterpriseNo','FieldType','DataType','Len','Description']] 59 cisco.to_csv('cisco_ie.csv') 60 iana_ie = pd.read_excel('./nf.xlsx',sheet_name="Sheet1") 61 iana_ie = iana_ie[(iana_ie['Val'].notnull()) & (iana_ie['Field Type'].notnull())] 62 regexp = re.compile(r'<.*') 63 iana_ie['FullID']=iana_ie['Val'].replace(regexp,'').astype(int) 64 iana_ie = iana_ie[iana_ie['FullID']<512] 65 iana_ie['ElementID'] = iana_ie['FullID'] 66 iana_ie['EnterpriseNo'] = 0 67 iana_ie['FieldType'] = iana_ie['Field Type'].apply(lambda x: x.replace('+','').replace(' ','_').replace('(','').replace(')','_').replace('%','').replace('|','')) 68 iana_ie['Len'] = iana_ie['Length'] 69 iana_ie = iana_ie[['FullID','ElementID','EnterpriseNo','FieldType','Len','Description']].reset_index(drop=True) 70 iana_ie_official = pd.read_csv('./ipfix-information-elements.csv') 71 iana_ie_official['DataType'] = iana_ie_official['Abstract Data Type'] 72 dtype = iana_ie_official[['ElementID','DataType']] 73 iana_result = iana_ie.join(dtype,on="ElementID",rsuffix='r')[['FullID','ElementID','EnterpriseNo','FieldType','DataType','Len','Description']] 74 iana_result.to_csv('iana_ie.csv') 75 iana_ie_official = pd.read_csv('./ipfix-information-elements.csv') 76 iana_ie_official['DataType'] = iana_ie_official['Abstract Data Type'] 77 iana_ie_official['FieldType']= iana_ie_official['Name'] 78 iana_ie_official['FullID'] = iana_ie_official['ElementID'] 79 iana_ie_official['EnterpriseNo'] = 0 80 iana_ie_official['Len'] = iana_ie_official['Units'] 81 iana_ie_official[['FullID','ElementID','EnterpriseNo','FieldType','DataType','Len','Description']].to_csv('iana_ie2.csv') 82