go.ligato.io/vpp-agent/v3@v3.5.0/tests/robot/libraries/kubernetes/KubeEnv.robot (about)

     1  *** Settings ***
     2  Documentation     This is a library to handle actions related to kubernetes cluster,
     3  ...               such as kubernetes setup  etc.
     4  ...
     5  ...               The code is aimed at few selected deployments:
     6  ...               a: 1-node 10-pods, nospecific applications, test use ping and nc to check connectivity.
     7  ...
     8  ...               This Resource manages the following suite variables:
     9  ...               ${testbed_connection} SSH connection index towards host in 1-node k8s cluster.
    10  ...               #${client_pod_name} client pod name assigned by k8s in 1-node 2-pod scenario.
    11  ...               #${server_pod_name} server pod name assigned by k8s in 1-node 2-pod scenario.
    12  Resource          ${CURDIR}/../all_libs.robot
    13  Resource          KubeCtl.robot
    14  Library           kube_parser.py
    15  
    16  *** Variables ***
    17  ${robot_root}                ${CURDIR}/../..
    18  ${ETCD_YAML_FILE_PATH}       ${robot_root}/resources/k8-yaml/etcd-k8.yaml
    19  ${SFC_YAML_FILE_PATH}        ${K8_GENERATED_CONFIG_FOLDER}/sfc.yaml
    20  ${VSWITCH_YAML_FILE_PATH}    ${K8_GENERATED_CONFIG_FOLDER}/vswitch.yaml
    21  ${VNF_YAML_FILE_PATH}        ${K8_GENERATED_CONFIG_FOLDER}/vnf.yaml
    22  ${NOVPP_YAML_FILE_PATH}      ${K8_GENERATED_CONFIG_FOLDER}/novpp.yaml
    23  ${PULL_IMAGES_PATH}          ${robot_root}/resources/k8-scripts/pull-images.sh
    24  
    25  ${POD_DEPLOY_APPEARS_TIMEOUT}    30s
    26  ${POD_REMOVE_DEFAULT_TIMEOUT}    60s
    27  
    28  *** Keywords ***
    29  Verify_All_Pods_Running
    30      [Arguments]    ${ssh_session}    ${excluded_pod_prefix}=invalid-pod-prefix-
    31      [Documentation]     Iterate over all pods of all namespaces (skipping \${excluded_pod_prefix} matches) and check running state.
    32      BuiltIn.Log_Many    ${ssh_session}    ${excluded_pod_prefix}
    33      ${all_pods_dict} =    KubeCtl.Get_Pods_All_Namespaces    ${ssh_session}
    34      ${pod_names} =    Collections.Get_Dictionary_Keys    ${all_pods_dict}
    35      : FOR    ${pod_name}   IN    @{pod_names}
    36      \     BuiltIn.Continue_For_Loop_If    """${excluded_pod_prefix}""" in """${pod_name}"""
    37      \     ${namesp} =    BuiltIn.Evaluate    &{all_pods_dict}[${pod_name}]['NAMESPACE']
    38      \     Verify_Pod_Running_And_Ready    ${ssh_session}    ${pod_name}    namespace=${namesp}
    39  
    40  Verify_K8s_Running
    41      [Arguments]    ${ssh_session}
    42      [Documentation]     We check for a particular (hardcoded) number of pods after init. Might be later replaced with
    43      ...    more detailed asserts.
    44      BuiltIn.Log_Many    ${ssh_session}
    45      BuiltIn.Comment    TODO: Make the expected number of pods configurable.
    46      ${all_pods_dict} =    KubeCtl.Get_Pods_All_Namespaces    ${ssh_session}
    47      BuiltIn.Length_Should_Be   ${all_pods_dict}     9
    48      Verify_All_Pods_Running    ${ssh_session}
    49  
    50  Get_Pod_Name_List_By_Prefix
    51      [Arguments]    ${ssh_session}    ${pod_prefix}
    52      [Documentation]    Get pods from all namespaces, parse with specified \${pod_prefix}, log and return the parsed result.
    53      BuiltIn.Log_Many    ${ssh_session}    ${pod_prefix}
    54      BuiltIn.Comment    TODO: Unify with Get_Pods or Get_Pods_All_Namespaces in KubeCtl.
    55      ${stdout} =    SshCommons.Switch_And_Execute_Command    ${ssh_session}    kubectl get pods
    56      ${output} =    kube_parser.parse_kubectl_get_pods_and_get_pod_name    ${stdout}    ${pod_prefix}
    57      Builtin.Log    ${output}
    58      [Return]    ${output}
    59  
    60  Deploy_Etcd_And_Verify_Running
    61      [Arguments]    ${ssh_session}    ${etcd_file}=${ETCD_YAML_FILE_PATH}
    62      [Documentation]     Deploy and verify ETCD pod and store its name.
    63      BuiltIn.Log_Many    ${ssh_session}    ${etcd_file}
    64      ${etcd_pod_name} =    Deploy_Pod_And_Verify_Running    ${ssh_session}    ${etcd_file}    etcd    timeout=${POD_DEPLOY_TIMEOUT}
    65      BuiltIn.Set_Suite_Variable    ${etcd_pod_name}
    66  
    67  Deploy_Vswitch_Pod_And_Verify_Running
    68      [Arguments]    ${ssh_session}    ${vswitch_file}=${VSWITCH_YAML_FILE_PATH}
    69      [Documentation]     Deploy and verify switch pod and store its name.
    70      BuiltIn.Log_Many    ${ssh_session}    ${vswitch_file}
    71      ${vswitch_pod_name} =    Deploy_Pod_And_Verify_Running    ${ssh_session}    ${vswitch_file}    vswitch-    timeout=${POD_DEPLOY_TIMEOUT}
    72      BuiltIn.Set_Suite_Variable    ${vswitch_pod_name}
    73  
    74  Deploy_SFC_Pod_And_Verify_Running
    75      [Arguments]    ${ssh_session}    ${sfc_file}=${SFC_YAML_FILE_PATH}
    76      [Documentation]     Deploy and verify switch pod and store its name.
    77      BuiltIn.Log_Many    ${ssh_session}    ${sfc_file}
    78      ${sfc_pod_name} =    Deploy_Pod_And_Verify_Running    ${ssh_session}    ${sfc_file}    sfc-    timeout=${POD_DEPLOY_TIMEOUT}
    79      BuiltIn.Set_Suite_Variable    ${sfc_pod_name}
    80  
    81  Deploy_VNF_Pods
    82      [Arguments]    ${ssh_session}    ${replicas}    ${cn-infra_file}=${VNF_YAML_FILE_PATH}
    83      [Documentation]     Deploy VNF pods, verify running and store their names.
    84      BuiltIn.Log_Many    ${ssh_session}    ${cn-infra_file}
    85      ${cn_infra_pod_name} =    Deploy_Multireplica_Pods_And_Verify_Running    ${ssh_session}    ${cn-infra_file}    vnf-    ${replicas}    namespace=default    timeout=${POD_DEPLOY_MULTIREPLICA_TIMEOUT}
    86      BuiltIn.Set_Suite_Variable    ${cn_infra_pod_name}
    87  
    88  Deploy_NoVPP_Pods
    89      [Arguments]    ${ssh_session}    ${replicas}    ${cn-infra_file}=${NOVPP_YAML_FILE_PATH}
    90      [Documentation]     Deploy NoVPP pods, verify running and store their names.
    91      BuiltIn.Log_Many    ${ssh_session}    ${cn-infra_file}
    92      ${cn_infra_pod_name} =    Deploy_Multireplica_Pods_And_Verify_Running    ${ssh_session}    ${cn-infra_file}    novpp-    ${replicas}    namespace=default    timeout=${POD_DEPLOY_MULTIREPLICA_TIMEOUT}
    93      BuiltIn.Set_Suite_Variable    ${cn_infra_pod_name}
    94  
    95  Remove_VSwitch_Pod_And_Verify_Removed
    96      [Arguments]    ${ssh_session}    ${vswitch_file}=${VSWITCH_YAML_FILE_PATH}
    97      [Documentation]    Execute delete commands, wait until  pod is removed.
    98      BuiltIn.Log_Many    ${ssh_session}    ${vswitch_file}
    99      KubeCtl.Delete_F    ${ssh_session}    ${vswitch_file}
   100      Wait_Until_Pod_Removed    ${ssh_session}    ${vswitch_pod_name}
   101  
   102  Remove_SFC_Pod_And_Verify_Removed
   103      [Arguments]    ${ssh_session}    ${sfc_file}=${SFC_YAML_FILE_PATH}
   104      [Documentation]    Execute delete commands, wait until  pod is removed.
   105      BuiltIn.Log_Many    ${ssh_session}    ${sfc_file}
   106      KubeCtl.Delete_F    ${ssh_session}    ${sfc_file}
   107      Wait_Until_Pod_Removed    ${ssh_session}    ${sfc_pod_name}
   108  
   109  Remove_Cn-Infra_Pod_And_Verify_Removed
   110      [Arguments]    ${ssh_session}    ${cn_infra_file}=${CN_INFRA_YAML_FILE_PATH}
   111      [Documentation]    Execute delete commands, wait until  pod is removed.
   112      BuiltIn.Log_Many    ${ssh_session}    ${cn_infra_file}
   113      KubeCtl.Delete_F    ${ssh_session}    ${cn_infra_file}
   114      Wait_Until_Pod_Removed    ${ssh_session}    ${cn_infra_pod_name}
   115  
   116  Remove_ETCD_Pod_And_Verify_Removed
   117      [Arguments]    ${ssh_session}    ${etcd_file}=${ETCD_YAML_FILE_PATH}
   118      [Documentation]    Execute delete commands, wait until  pod is removed.
   119      BuiltIn.Log_Many    ${ssh_session}    ${etcd_file}
   120      KubeCtl.Delete_F    ${ssh_session}    ${etcd_file}
   121      Wait_Until_Pod_Removed    ${ssh_session}    ${etcd_pod_name}
   122  
   123  Verify_Multireplica_Pods_Running
   124      [Arguments]    ${ssh_session}    ${pod_prefix}    ${nr_replicas}    ${namespace}
   125      [Documentation]     Check there is expected number of pods and they are running.
   126      BuiltIn.Log_Many    ${ssh_session}    ${pod_prefix}    ${nr_replicas}    ${namespace}
   127      BuiltIn.Comment    TODO: Join single- and multi- replica keywords.
   128      ${pods_list} =    Get_Pod_Name_List_By_Prefix    ${ssh_session}    ${pod_prefix}
   129      BuiltIn.Length_Should_Be   ${pods_list}     ${nr_replicas}
   130      : FOR    ${pod_name}    IN    @{pods_list}
   131      \    Verify_Pod_Running_And_Ready    ${ssh_session}    ${pod_name}    namespace= ${namespace}
   132      BuiltIn.Return_From_Keyword    ${pods_list}
   133  
   134  Deploy_Multireplica_Pods_And_Verify_Running
   135      [Arguments]    ${ssh_session}    ${pod_file}    ${pod_prefix}    ${nr_replicas}    ${namespace}=default    ${timeout}=${POD_DEPLOY_MULTIREPLICA_TIMEOUT}
   136      [Documentation]     Apply the provided yaml file with more replica specified, wait until pods are running, return pods details.
   137      BuiltIn.Log_Many    ${ssh_session}    ${pod_file}    ${pod_prefix}    ${nr_replicas}    ${namespace}    ${timeout}
   138      BuiltIn.Comment    TODO: Join single- and multi- replica keywords.
   139      KubeCtl.Apply_F    ${ssh_session}    ${pod_file}
   140      ${pods_details} =    BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}   4s    Verify_Multireplica_Pods_Running    ${ssh_session}    ${pod_prefix}    ${nr_replicas}    ${namespace}
   141      [Return]    ${pods_details}
   142  
   143  Verify_Multireplica_Pods_Removed
   144      [Arguments]    ${ssh_session}    ${pod_prefix}
   145      [Documentation]     Check no pods are running with prefix: ${pod_prefix}
   146      BuiltIn.Log_Many    ${ssh_session}    ${pod_prefix}
   147      BuiltIn.Comment    TODO: Join single- and multi- replica keywords.
   148      ${pods_list} =    Get_Pod_Name_List_By_Prefix    ${ssh_session}    ${pod_prefix}
   149      BuiltIn.Length_Should_Be   ${pods_list}     0
   150  
   151  Remove_Multireplica_Pods_And_Verify_Removed
   152      [Arguments]    ${ssh_session}    ${pod_file}    ${pod_prefix}
   153      [Documentation]     Remove pods and verify they are removed.
   154      BuiltIn.Log_Many    ${ssh_session}    ${pod_file}    ${pod_prefix}
   155      KubeCtl.Delete_F    ${ssh_session}    ${pod_file}
   156      BuiltIn.Wait_Until_Keyword_Succeeds    ${POD_REMOVE_MULTIREPLICA_TIMEOUT}    5s    Verify_Multireplica_Pods_Removed    ${ssh_session}    ${pod_prefix}
   157  
   158  Remove_NonVPP_Pod_And_Verify_Removed
   159      [Arguments]    ${ssh_session}    ${nginx_file}=${NGINX_POD_FILE}
   160      [Documentation]    Remove pod and verify removal, nginx being the default file.
   161      BuiltIn.Log_Many    ${ssh_session}    ${nginx_file}
   162      KubeCtl.Delete_F    ${ssh_session}    ${nginx_file}
   163      Wait_Until_Pod_Removed    ${ssh_session}    ${nginx_pod_name}
   164  
   165  Get_Deployed_Pod_Name
   166      [Arguments]    ${ssh_session}    ${pod_prefix}
   167      [Documentation]    Get list of pod names matching the prefix, check there is just one, return the name.
   168      BuiltIn.Log_Many    ${ssh_session}    ${pod_prefix}
   169      ${pod_name_list} =   Get_Pod_Name_List_By_Prefix    ${ssh_session}    ${pod_prefix}
   170      BuiltIn.Length_Should_Be    ${pod_name_list}    1
   171      ${pod_name} =    BuiltIn.Evaluate     ${pod_name_list}[0]
   172      [Return]    ${pod_name}
   173  
   174  Deploy_Pod_And_Verify_Running
   175      [Arguments]    ${ssh_session}    ${pod_file}    ${pod_prefix}    ${timeout}=${POD_DEPLOY_DEFAULT_TIMEOUT}
   176      [Documentation]    Deploy pod defined by \${pod_file}, wait until a pod matching \${pod_prefix} appears, check it was only 1 such pod, extract its name, wait until it is running, log and return the name.
   177      Builtin.Log_Many    ${ssh_session}    ${pod_file}    ${pod_prefix}
   178      KubeCtl.Apply_F    ${ssh_session}    ${pod_file}
   179      ${pod_name} =    BuiltIn.Wait_Until_Keyword_Succeeds    ${POD_DEPLOY_APPEARS_TIMEOUT}    2s    Get_Deployed_Pod_Name    ${ssh_session}    ${pod_prefix}
   180      Wait_Until_Pod_Running    ${ssh_session}    ${pod_name}    timeout=${timeout}
   181      BuiltIn.Log    ${pod_name}
   182      [Return]    ${pod_name}
   183  
   184  Remove_Pod_And_Verify_Removed
   185      [Arguments]    ${ssh_session}    ${pod_file}    ${pod_name}
   186      [Documentation]    Remove pod defined by \${pod_file}, wait for \${pod_name} to get removed.
   187      BuiltIn.Log_Many    ${ssh_session}    ${pod_file}    ${pod_name}
   188      KubeCtl.Delete_F    ${ssh_session}    ${pod_file}
   189      Wait_Until_Pod_Removed    ${ssh_session}    ${pod_name}
   190  
   191  Verify_Pod_Not_Terminating
   192      [Arguments]    ${ssh_session}    ${pod_name}    ${namespace}=default
   193      [Documentation]    Get pods of \${namespace}, parse status of \${pod_name}, check it is not Terminating.
   194      BuiltIn.Log_Many    ${ssh_session}    ${pod_name}    ${namespace}
   195      &{pods} =     KubeCtl.Get_Pods    ${ssh_session}    namespace=${namespace}
   196      Return From Keyword If    "${pod_name}" not in ${pods}.keys()
   197      ${status} =    BuiltIn.Evaluate    &{pods}[${pod_name}]['STATUS']
   198      BuiltIn.Should_Not_Be_Equal_As_Strings    ${status}    Terminating
   199  
   200  Verify_Pod_Running_And_Ready
   201      [Arguments]    ${ssh_session}    ${pod_name}    ${namespace}=default
   202      [Documentation]    Get pods of \${namespace}, parse status of \${pod_name}, check it is Running, parse for ready containes of \${pod_name}, check it is all of them.
   203      BuiltIn.Log_Many    ${ssh_session}    ${pod_name}    ${namespace}
   204      &{pods} =     KubeCtl.Get_Pods    ${ssh_session}    namespace=${namespace}
   205      ${status} =    BuiltIn.Evaluate    &{pods}[${pod_name}]['STATUS']
   206      BuiltIn.Should_Be_Equal_As_Strings    ${status}    Running
   207      ${ready} =    BuiltIn.Evaluate    &{pods}[${pod_name}]['READY']
   208      ${ready_containers}    ${out_of_containers} =    String.Split_String    ${ready}    separator=${/}    max_split=1
   209      BuiltIn.Should_Be_Equal_As_Strings    ${ready_containers}    ${out_of_containers}
   210  
   211  Wait_Until_Pod_Running
   212      [Arguments]    ${ssh_session}    ${pod_name}    ${timeout}=${POD_RUNNING_DEFAULT_TIMEOUT}    ${check_period}=5s    ${namespace}=default
   213      [Documentation]    WUKS around Verify_Pod_Running_And_Ready.
   214      BuiltIn.Log_Many    ${ssh_session}    ${pod_name}    ${timeout}    ${check_period}    ${namespace}
   215      BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    ${check_period}    Verify_Pod_Running_And_Ready    ${ssh_session}    ${pod_name}    namespace=${namespace}
   216  
   217  Verify_Pod_Not_Present
   218      [Arguments]    ${ssh_session}    ${pod_name}=${NONE}    ${namespace}=default
   219      [Documentation]    Get pods for \${namespace}, check \${pod_name} is not one of them.
   220      BuiltIn.Log_Many    ${ssh_session}    ${pod_name}    ${namespace}
   221      ${pods} =     KubeCtl.Get_Pods    ${ssh_session}    namespace=${namespace}
   222      BuiltIn.Run_Keyword_If    "${pod_name}" == "${NONE}"    BuiltIn.Should_Be_Empty    ${pods}
   223      ...    ELSE    Collections.Dictionary_Should_Not_Contain_Key     ${pods}    ${pod_name}
   224  
   225  Wait_Until_Pod_Removed
   226      [Arguments]    ${ssh_session}    ${pod_name}=${NONE}    ${timeout}=${POD_REMOVE_DEFAULT_TIMEOUT}    ${check_period}=5s    ${namespace}=default
   227      [Documentation]    WUKS around Verify_Pod_Not_Present.
   228      BuiltIn.Log_Many    ${ssh_session}    ${pod_name}    ${timeout}    ${check_period}    ${namespace}
   229      BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    ${check_period}    Verify_Pod_Not_Present    ${ssh_session}    ${pod_name}    namespace=${namespace}
   230  
   231  Run_Command_In_Pod
   232      [Arguments]    ${command}    ${pod_name}
   233      [Documentation]    Execute command on the pod, log and return retval, stdout, stderr.
   234      BuiltIn.Log_Many    ${command}     ${pod_name}
   235      BuiltIn.Comment    TODO: Do not mention pods and move to SshCommons.robot or similar.
   236      SSHLibrary.Switch Connection    ${testbed_connection}
   237      ${output} =    SSHLibrary.Execute Command    kubectl exec -it ${pod_name} -- ${command}    return_stdout=True    return_stderr=True    return_rc=True
   238      BuiltIn.Should_Be_Equal_As_integers    ${output[2]}    ${0}
   239      SshCommons.Append_Command_Log    ${command}    ${output}
   240      [Return]    ${output[0]}
   241  
   242  Init_Infinite_Command_In_Pod
   243      [Arguments]    ${command}    ${ssh_session}=${EMPTY}    ${prompt}=${EMPTY}
   244      [Documentation]    Switch if \${ssh_session}, configure if \${prompt}, write \${command}.
   245      BuiltIn.Log_Many    ${command}    ${ssh_session}    ${prompt}
   246      BuiltIn.Comment    TODO: Do not mention pods and move to SshCommons.robot or similar.
   247      BuiltIn.Run_Keyword_If    """${ssh_session}""" != """${EMPTY}"""     SSHLibrary.Switch_Connection    ${ssh_session}
   248      BuiltIn.Run_Keyword_If    """${prompt}""" != """${EMPTY}"""    SSHLibrary.Set_Client_Configuration    prompt=${prompt}
   249      SSHLibrary.Write    ${command}
   250      SshCommons.Append_Command_Log    ${command}
   251  
   252  Stop_Infinite_Command_In_Pod
   253      [Arguments]    ${ssh_session}=${EMPTY}     ${prompt}=${EMPTY}
   254      [Documentation]    Switch if \${ssh_session}, configure if \${prompt}, write ctrl+c, read until prompt, log and return output.
   255      BuiltIn.Log_Many    ${ssh_session}    ${prompt}
   256      BuiltIn.Comment    TODO: Do not mention pods and move to SshCommons.robot or similar.
   257      BuiltIn.Run_Keyword_If    """${ssh_session}""" != """${EMPTY}"""     SSHLibrary.Switch_Connection    ${ssh_session}
   258      BuiltIn.Run_Keyword_If    """${prompt}""" != """${EMPTY}"""    SSHLibrary.Set_Client_Configuration    prompt=${prompt}
   259      Write_Bare_Ctrl_C
   260      ${output1} =     SSHLibrary.Read_Until    ^C
   261      ${output2} =     SSHLibrary.Read_Until_Prompt
   262      BuiltIn.Log_Many     ${output1}    ${output2}
   263      ${output} =    Builtin.Set_Variable    ${output1}${output2}
   264      SshCommons.Append_Command_Log    ^C    ${output}
   265      [Return]    ${output}
   266  
   267  Write_Bare_Ctrl_C
   268      [Documentation]    Construct ctrl+c character and SSH-write it (without endline) to the current SSH connection.
   269      ...    Do not read anything yet.
   270      BuiltIn.Comment    TODO: Move to SshCommons.robot or similar.
   271      ${ctrl_c} =    BuiltIn.Evaluate    chr(int(3))
   272      SSHLibrary.Write_Bare    ${ctrl_c}
   273  
   274  Get_Into_Container_Prompt_In_Pod
   275      [Arguments]    ${ssh_session}    ${pod_name}    ${prompt}=${EMPTY}
   276      [Documentation]    Configure if prompt, execute interactive bash in ${pod_name}, read until prompt, log and return output.
   277      BuiltIn.Log_Many    ${ssh_session}    ${pod_name}    ${prompt}
   278      # TODO: PodBash.robot?
   279      SSHLibrary.Switch_Connection    ${ssh_session}
   280      SSHLibrary.Set_Client_Configuration    prompt=${prompt}
   281      ${command} =    BuiltIn.Set_Variable    kubectl exec -i -t ${pod_name} /bin/sh
   282      SSHLibrary.Write    ${command}
   283      ${output} =     SSHLibrary.Read_Until_Prompt
   284      SshCommons.Append_Command_Log    ${command}    ${output}
   285      [Return]    ${output}
   286  
   287  Leave_Container_Prompt_In_Pod
   288      [Arguments]     ${ssh_session}    ${prompt}=$
   289      [Documentation]    Configure prompt, send ctrl+c, write "exit", read until prompt, log and return output.
   290      BuiltIn.Log_Many    ${ssh_session}    ${prompt}
   291      # TODO: PodBash.robot?
   292      SSHLibrary.Switch_Connection    ${ssh_session}
   293      SSHLibrary.Set_Client_Configuration    prompt=${prompt}
   294      Write_Bare_Ctrl_C
   295      SSHLibrary.Write    exit
   296      ${output} =     SSHLibrary.Read_Until_Prompt
   297      SshCommons.Append_Command_Log    ^Cexit    ${output}
   298      [Return]    ${output}
   299  
   300  Verify_Cluster_Node_Ready
   301      [Arguments]    ${ssh_session}    ${node_name}
   302      [Documentation]    Get nodes, parse status of \${node_name}, check it is Ready, return nodes.
   303      BuiltIn.Log_Many    ${ssh_session}    ${node_name}
   304      BuiltIn.Comment    FIXME: Avoid repeated get_nodes when called from Verify_Cluster_Ready.
   305      ${nodes} =    KubeCtl.Get_Nodes    ${ssh_session}
   306      BuiltIn.Log    ${nodes}
   307      ${status} =    BuiltIn.Evaluate    &{nodes}[${node_name}]['STATUS']
   308      BuiltIn.Should_Be_Equal    ${status}    Ready
   309      [Return]    ${nodes}
   310  
   311  Verify_Cluster_Ready
   312      [Arguments]     ${ssh_session}    ${nr_nodes}
   313      [Documentation]    Get nodes, check there are \${nr_nodes}, for each node Verify_Cluster_Node_Ready.
   314      BuiltIn.Log_Many     ${ssh_session}    ${nr_nodes}
   315      ${nodes} =    KubeCtl.Get_Nodes    ${ssh_session}
   316      BuiltIn.Log    ${nodes}
   317      BuiltIn.Length_Should_Be    ${nodes}    ${nr_nodes}
   318      ${names} =     Collections.Get_Dictionary_Keys     ${nodes}
   319      : FOR    ${name}    IN    @{names}
   320      \    Verify_Cluster_Node_Ready    ${ssh_session}    ${name}
   321  
   322  Wait_Until_Cluster_Ready
   323      [Arguments]    ${ssh_session}    ${nr_nodes}    ${timeout}=${CLUSTER_READY_TIMEOUT}    ${check_period}=5s
   324      [Documentation]    WUKS around Verify_Cluster_Ready.
   325      BuiltIn.Log_Many    ${ssh_session}    ${nr_nodes}    ${timeout}    ${check_period}
   326      BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    ${check_period}    Verify_Cluster_Ready    ${ssh_session}    ${nr_nodes}
   327  
   328  Log_Etcd
   329      [Arguments]    ${ssh_session}
   330      [Documentation]    Check there is exactly one etcd pod, get its logs
   331      ...    (and do nothing with them, except the implicit Log).
   332      Builtin.Log_Many    ${ssh_session}
   333      ${pod_list} =    Get_Pod_Name_List_By_Prefix    ${ssh_session}    etcd
   334      BuiltIn.Log    ${pod_list}
   335      BuiltIn.Length_Should_Be    ${pod_list}    1
   336      KubeCtl.Logs    ${ssh_session}    @{pod_list}[0]    namespace=default
   337  
   338  Log_Vswitch
   339      [Arguments]    ${ssh_session}    ${exp_nr_vswitch}=${K8_CLUSTER_${CLUSTER_ID}_NODES}
   340      [Documentation]    Check there is expected number of vswitch pods, get logs from them an cn-infra containers
   341      ...    (and do nothing except the implicit Log).
   342      Builtin.Log_Many    ${ssh_session}    ${exp_nr_vswitch}
   343      ${pod_list} =    Get_Pod_Name_List_By_Prefix    ${ssh_session}    vswitch
   344      BuiltIn.Log    ${pod_list}
   345      BuiltIn.Length_Should_Be    ${pod_list}    ${exp_nr_vswitch}
   346      : FOR    ${vswitch_pod}    IN    @{pod_list}
   347      \    KubeCtl.Logs    ${ssh_session}    ${vswitch_pod}    namespace=default
   348      \    Run Command In Pod    vppctl show int    ${vswitch_pod}
   349      \    Run Command In Pod    vppctl show int address   ${vswitch_pod}
   350      \    Run Command In Pod    vppctl show errors    ${vswitch_pod}
   351  
   352  Log_Pods_For_Debug
   353      [Arguments]    ${ssh_session}    ${exp_nr_vswitch}=${K8_CLUSTER_${CLUSTER_ID}_NODES}
   354      [Documentation]    Call multiple keywords to get various logs
   355      ...    (and do nothing with them, except the implicit Log).
   356      Builtin.Log_Many    ${ssh_session}    ${exp_nr_vswitch}
   357      Log_Etcd    ${ssh_session}
   358      Log_Vswitch    ${ssh_session}    ${exp_nr_vswitch}
   359      :FOR    ${vnf_index}    IN RANGE    ${vnf_count}
   360      \    Run Command In Pod    vppctl show int              vnf-vpp-${vnf_index}
   361      \    Run Command In Pod    vppctl show int address      vnf-vpp-${vnf_index}
   362      \    Run Command In Pod    vppctl show errors           vnf-vpp-${vnf_index}
   363      :FOR    ${novpp_index}    IN RANGE    ${novpp_count}
   364      \    Run Command In Pod    ip link        novpp-${novpp_index}
   365      \    Run Command In Pod    ip address     novpp-${novpp_index}
   366      \    Run Command In Pod    ip neighbor    novpp-${novpp_index}
   367  
   368  Open_Connection_To_Node
   369      [Arguments]    ${name}    ${cluster_id}    ${node_index}
   370      BuiltIn.Log_Many    ${name}    ${node_index}
   371      ${connection}=    SshCommons.Open_Ssh_Connection_Kube    ${name}    ${K8_CLUSTER_${cluster_id}_VM_${node_index}_PUBLIC_IP}    ${K8_CLUSTER_${cluster_id}_VM_${node_index}_USER}    ${K8_CLUSTER_${cluster_id}_VM_${node_index}_PSWD}
   372      [Return]    ${connection}