github.com/zartbot/goflow@v0.0.0-20221001080520-456f7dfad288/tools/parse_ipfix.py (about)

     1  import pandas as pd
     2  import re
     3  cisco = pd.read_excel('./nf.xlsx',sheet_name="Sheet2")
     4  cisco = cisco[(cisco['Val'].notnull()) & (cisco['Field Type'].notnull())]
     5  regexp = re.compile(r'\(.*\)')
     6  cisco['FullID']=cisco['Val'].replace(regexp,'').astype(int)
     7  cisco = cisco[cisco['FullID']<65000]
     8  cisco['ElementID'] = cisco['FullID'] - 32768
     9  cisco['EnterpriseNo'] = 9
    10  cisco['FieldType'] = cisco['Field Type'].apply(lambda x: x.replace('+','').replace(' ','_').replace('(','').replace(')','_').replace('%',''))
    11  cisco = cisco[['FullID','ElementID','EnterpriseNo','FieldType','Len','Description']].reset_index(drop=True)
    12  cisco['Description'].fillna('',inplace=True)
    13  cisco['Len'] = cisco['Len'].apply(lambda x: str(x).replace('\n','none'))
    14  #cisco['DataType'] = 'octetArray'
    15  cisco.loc[cisco['FieldType'].str.contains('IPv4Address',regex=False),'DataType'] = 'ipv4Address'
    16  cisco.loc[cisco['FieldType'].str.contains('IPv6Address',regex=False),'DataType'] = 'ipv6Address'
    17  cisco.loc[cisco['FieldType'].str.contains('AddressIPv4',regex=False),'DataType'] = 'ipv4Address'
    18  cisco.loc[cisco['FieldType'].str.contains('AddressIPv6',regex=False),'DataType'] = 'ipv6Address'
    19  cisco.loc[(cisco['Description'].str.contains('addr',case=False) & cisco['Description'].str.contains('v4',case=False)) ,'DataType'] = 'ipv4Address'
    20  cisco.loc[(cisco['Description'].str.contains('addr',case=False) & cisco['Description'].str.contains('v6',case=False)) ,'DataType'] = 'ipv6Address'
    21  
    22  cisco.loc[cisco['FieldType'].str.contains('uri',case=False,regex=False),'DataType'] = 'string'
    23  cisco.loc[cisco['FieldType'].str.contains('string',case=False,regex=False),'DataType'] = 'string'
    24  cisco.loc[cisco['Description'].str.contains('string',case=False),'DataType'] = 'string'
    25  cisco.loc[cisco['Len'].str.contains('string',case=False),'DataType'] = 'string'
    26  cisco.loc[cisco['Len'].str.contains('Bitstring',case=False),'DataType'] = 'octetArray'
    27  cisco.loc[cisco['Len'].str.contains('u8',case=False),'DataType'] = 'unsigned8'
    28  cisco.loc[cisco['Len'].str.contains('u16',case=False),'DataType'] = 'unsigned16'
    29  cisco.loc[cisco['Len'].str.contains('u32',case=False),'DataType'] = 'unsigned32'
    30  cisco.loc[cisco['Len'].str.contains('u64',case=False),'DataType'] = 'unsigned64'
    31  cisco.loc[cisco['FieldType'].str.contains('Port'),'DataType'] ='unsigned16'
    32  cisco.loc[cisco['FieldType'].str.contains('Port',case=False) & (cisco['Len'] == '2'),'DataType'] = 'unsigned16'
    33  cisco.loc[ (cisco['FieldType'].str.contains('count',case=False) &  (cisco['Len'] == '1')),'DataType'] = 'unsigned8'
    34  cisco.loc[ (cisco['FieldType'].str.contains('count',case=False) &  (cisco['Len'] == '2')),'DataType'] = 'unsigned16'
    35  cisco.loc[ (cisco['FieldType'].str.contains('count',case=False) &  (cisco['Len'] == '4')),'DataType'] = 'unsigned32'
    36  cisco.loc[ (cisco['FieldType'].str.contains('count',case=False) &  (cisco['Len'] == '8')),'DataType'] = 'unsigned64'
    37  cisco.loc[(cisco['Description'].str.contains('number',case=False) &  (cisco['Len'] == '1')),'DataType'] = 'unsigned8'
    38  cisco.loc[(cisco['Description'].str.contains('number',case=False) &  (cisco['Len'] == '2')),'DataType'] = 'unsigned16'
    39  cisco.loc[(cisco['Description'].str.contains('number',case=False) &  (cisco['Len'] == '4')),'DataType'] = 'unsigned32'
    40  cisco.loc[(cisco['Description'].str.contains('number',case=False) &  (cisco['Len'] == '8')),'DataType'] = 'unsigned64'
    41  cisco.loc[(cisco['Description'].str.contains('number',case=False) &  (cisco['Len'] == '4 or 8')),'DataType'] = 'unsigned64'
    42  cisco.loc[(cisco['Description'].str.contains('msec',case=False)),'DataType'] = 'dateTimeMilliseconds'
    43  cisco.loc[(cisco['Description'].str.contains('id',case=False)&  (cisco['Len'] == '1')),'DataType'] = 'unsigned8'
    44  cisco.loc[(cisco['Description'].str.contains('id',case=False)&  (cisco['Len'] == '2')),'DataType'] = 'unsigned16'
    45  cisco.loc[(cisco['Description'].str.contains('id',case=False)&  (cisco['Len'] == '4')),'DataType'] = 'unsigned32'
    46  cisco.loc[(cisco['Description'].str.contains('id',case=False)&  (cisco['Len'] == '8')),'DataType'] = 'unsigned64'
    47  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '1')),'DataType'] = 'unsigned8'
    48  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '2')),'DataType'] = 'unsigned16'
    49  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '4')),'DataType'] = 'unsigned32'
    50  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '8')),'DataType'] = 'unsigned64'
    51  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '32bits')),'DataType'] = 'unsigned32'
    52  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == '64bits')),'DataType'] = 'unsigned64'
    53  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == 'IPV4')),'DataType'] = 'ipv4Address'
    54  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == 'IPv4')),'DataType'] = 'ipv4Address'
    55  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == 'IPV6')),'DataType'] = 'ipv6Address'
    56  cisco.loc[(cisco['DataType'].isna()&(cisco['Len'] == 'IPv6')),'DataType'] = 'ipv6Address'
    57  cisco.loc[(cisco['DataType'].isna()),'DataType'] = 'octetArray'
    58  cisco = cisco[['FullID','ElementID','EnterpriseNo','FieldType','DataType','Len','Description']]
    59  cisco.to_csv('cisco_ie.csv')
    60  iana_ie = pd.read_excel('./nf.xlsx',sheet_name="Sheet1")
    61  iana_ie = iana_ie[(iana_ie['Val'].notnull()) & (iana_ie['Field Type'].notnull())]
    62  regexp = re.compile(r'<.*')
    63  iana_ie['FullID']=iana_ie['Val'].replace(regexp,'').astype(int)
    64  iana_ie = iana_ie[iana_ie['FullID']<512]
    65  iana_ie['ElementID'] = iana_ie['FullID'] 
    66  iana_ie['EnterpriseNo'] = 0
    67  iana_ie['FieldType'] = iana_ie['Field Type'].apply(lambda x: x.replace('+','').replace(' ','_').replace('(','').replace(')','_').replace('%','').replace('|',''))
    68  iana_ie['Len'] = iana_ie['Length']
    69  iana_ie = iana_ie[['FullID','ElementID','EnterpriseNo','FieldType','Len','Description']].reset_index(drop=True)
    70  iana_ie_official = pd.read_csv('./ipfix-information-elements.csv')
    71  iana_ie_official['DataType'] = iana_ie_official['Abstract Data Type'] 
    72  dtype = iana_ie_official[['ElementID','DataType']]
    73  iana_result = iana_ie.join(dtype,on="ElementID",rsuffix='r')[['FullID','ElementID','EnterpriseNo','FieldType','DataType','Len','Description']]
    74  iana_result.to_csv('iana_ie.csv')
    75  iana_ie_official = pd.read_csv('./ipfix-information-elements.csv')
    76  iana_ie_official['DataType'] = iana_ie_official['Abstract Data Type'] 
    77  iana_ie_official['FieldType']= iana_ie_official['Name']
    78  iana_ie_official['FullID'] =  iana_ie_official['ElementID']
    79  iana_ie_official['EnterpriseNo'] = 0
    80  iana_ie_official['Len'] =  iana_ie_official['Units']
    81  iana_ie_official[['FullID','ElementID','EnterpriseNo','FieldType','DataType','Len','Description']].to_csv('iana_ie2.csv')
    82