github.com/kubearmor/cilium@v1.6.12/bugtool/cmd/configuration.go (about) 1 // Copyright 2017-2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package cmd 16 17 import ( 18 "encoding/json" 19 "fmt" 20 "io/ioutil" 21 "os" 22 "path/filepath" 23 "strings" 24 25 "github.com/cilium/cilium/pkg/components" 26 "github.com/cilium/cilium/pkg/defaults" 27 ) 28 29 // BugtoolConfiguration creates and loads the configuration file used to run 30 // the commands. The only commands not managed by the configuration is initial 31 // setup, for ex. searching for Cilium daemonset pods or running uname. 32 type BugtoolConfiguration struct { 33 // Commands is the exact commands that will be run by the bugtool 34 Commands []string `json:"commands"` 35 } 36 37 func setupDefaultConfig(path string, k8sPods []string, confDir, cmdDir string) (*BugtoolConfiguration, error) { 38 c := BugtoolConfiguration{defaultCommands(confDir, cmdDir, k8sPods)} 39 return &c, save(&c, path) 40 } 41 42 func defaultCommands(confDir string, cmdDir string, k8sPods []string) []string { 43 var commands []string 44 // Not expecting all of the commands to be available 45 commands = []string{ 46 // Host and misc 47 "ps auxfw", 48 "hostname", 49 "ip a", 50 "ip -4 r", 51 "ip -6 r", 52 "ip -d -s l", 53 "ip -4 n", 54 "ip -6 n", 55 "ss -t -p -a -i -s", 56 "ss -u -p -a -i -s", 57 "nstat", 58 "uname -a", 59 "dig", 60 "netstat -a", 61 "pidstat", 62 "arp", 63 "top -b -n 1", 64 "uptime", 65 "dmesg --time-format=iso", 66 "bpftool map show", 67 "bpftool prog show", 68 // LB and CT map for debugging services; using bpftool for a reliable dump 69 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb4_services_v2", 70 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb4_services", 71 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb4_backends", 72 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb4_reverse_nat", 73 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_ct4_global", 74 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_ct_any4_global", 75 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb6_services_v2", 76 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb6_services", 77 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb6_backends", 78 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb6_reverse_nat", 79 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_ct6_global", 80 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_ct_any6_global", 81 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_snat_v4_external", 82 "bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_snat_v6_external", 83 // Versions 84 "docker version", 85 "docker info", 86 // Docker and Kubernetes logs from systemd 87 "journalctl -u cilium*", 88 "journalctl -u kubelet", 89 // iptables 90 "iptables-save -c", 91 "iptables -S", 92 "ip6tables -S", 93 "iptables -L -v", 94 "ip rule", 95 "ip -4 route show table 2005", 96 "ip -6 route show table 2005", 97 "ip -4 route show table 200", 98 "ip -6 route show table 200", 99 // xfrm 100 "ip xfrm policy", 101 "ip -s xfrm state | awk '!/auth|enc|aead|auth-trunc|comp/'", 102 // gops 103 fmt.Sprintf("gops memstats $(pidof %s)", components.CiliumAgentName), 104 fmt.Sprintf("gops stack $(pidof %s)", components.CiliumAgentName), 105 fmt.Sprintf("gops stats $(pidof %s)", components.CiliumAgentName), 106 // Get list of open file descriptors managed by the agent 107 fmt.Sprintf("ls -la /proc/$(pidof %s)/fd", components.CiliumAgentName), 108 } 109 110 // Commands that require variables and / or more configuration are added 111 // separately below 112 commands = append(commands, catCommands()...) 113 commands = append(commands, ethoolCommands()...) 114 commands = append(commands, copyConfigCommands(confDir, k8sPods)...) 115 commands = append(commands, copyCiliumInfoCommands(cmdDir, k8sPods)...) 116 117 return k8sCommands(commands, k8sPods) 118 } 119 120 func save(c *BugtoolConfiguration, path string) error { 121 f, err := os.Create(path) 122 if err != nil { 123 return fmt.Errorf("Failed to open file %s for writing: %s", path, err) 124 } 125 defer f.Close() 126 127 data, err := json.MarshalIndent(c, "", "\t") 128 if err != nil { 129 return fmt.Errorf("Cannot marshal config %s", err) 130 } 131 err = ioutil.WriteFile(path, data, 0644) 132 if err != nil { 133 return fmt.Errorf("Cannot write config %s", err) 134 } 135 return nil 136 } 137 138 func loadConfigFile(path string) (*BugtoolConfiguration, error) { 139 var content []byte 140 var err error 141 content, err = ioutil.ReadFile(path) 142 143 if err != nil { 144 return nil, err 145 } 146 147 var c BugtoolConfiguration 148 err = json.Unmarshal(content, &c) 149 return &c, err 150 } 151 152 func catCommands() []string { 153 files := []string{ 154 "/proc/net/xfrm_stat", 155 "/proc/sys/net/core/bpf_jit_enable", 156 "/proc/kallsyms", 157 "/etc/resolv.conf", 158 "/var/log/docker.log", 159 "/var/log/daemon.log", 160 "/var/log/messages", 161 } 162 // Only print the files that do exist to reduce number of errors in 163 // archive 164 commands := []string{} 165 for _, f := range files { 166 if _, err := os.Stat(f); os.IsNotExist(err) { 167 continue 168 } 169 commands = append(commands, fmt.Sprintf("cat %s", f)) 170 } 171 // TODO: handle K8s case as well. 172 return commands 173 } 174 175 func copyConfigCommands(confDir string, k8sPods []string) []string { 176 commands := []string{} 177 // Location is a convenience structure to avoid too many long lines 178 type Location struct { 179 Src string 180 Dst string 181 } 182 183 // These locations don't depend on the kernel version for running so we 184 // can add them in this scope. 185 locations := []Location{ 186 {"/proc/config", fmt.Sprintf("%s/kernel-config", confDir)}, 187 {"/proc/config.gz", fmt.Sprintf("%s/kernel-config.gz", confDir)}, 188 } 189 190 // The following lines copy the kernel configuration. This code is 191 // duplicated for the non Kubernetes case. The variables preventing 192 // them to be one block is the pod prefix and namespace used in the 193 // path. This should be refactored. 194 if len(k8sPods) == 0 { 195 kernel, _ := execCommand("uname -r") 196 kernel = strings.TrimSpace(kernel) 197 // Append the boot config for the current kernel 198 l := Location{fmt.Sprintf("/boot/config-%s", kernel), 199 fmt.Sprintf("%s/kernel-config-%s", confDir, kernel)} 200 locations = append(locations, l) 201 202 // Use the locations to create command strings 203 for _, location := range locations { 204 if _, err := os.Stat(location.Src); os.IsNotExist(err) { 205 continue 206 } 207 commands = append(commands, fmt.Sprintf("cp %s %s", location.Src, location.Dst)) 208 } 209 } else { 210 // If there are multiple pods, we want to get all of the kernel 211 // configs. Therefore we need copy commands for all the pods. 212 for _, pod := range k8sPods { 213 prompt := podPrefix(pod, "uname -r") 214 kernel, _ := execCommand(prompt) 215 kernel = strings.TrimSpace(kernel) 216 l := Location{fmt.Sprintf("/boot/config-%s", kernel), 217 fmt.Sprintf("%s/kernel-config-%s", confDir, kernel)} 218 locations = append(locations, l) 219 220 // The location is mostly the same but the command is 221 // prepended with 'kubectl` and the path contains the 222 // namespace and pod. For ex: 223 // kubectl cp kube-system/cilium-kg8lv:/tmp/cilium-bugtool-243785589.tar /tmp/cilium-bugtool-243785589.tar 224 for _, location := range locations { 225 kubectlArg := fmt.Sprintf("%s/%s:%s", k8sNamespace, pod, location.Src) 226 cmd := fmt.Sprintf("%s %s %s %s", "kubectl", "cp", kubectlArg, location.Dst) 227 commands = append(commands, cmd) 228 } 229 } 230 } 231 return commands 232 } 233 234 func copyCiliumInfoCommands(cmdDir string, k8sPods []string) []string { 235 // Most of the output should come via debuginfo but also adding 236 // these ones for skimming purposes 237 ciliumCommands := []string{ 238 fmt.Sprintf("cilium debuginfo --output=markdown,json -f --output-directory=%s", cmdDir), 239 "cilium metrics list", 240 "cilium fqdn cache list", 241 "cilium config", 242 "cilium bpf tunnel list", 243 "cilium bpf lb list", 244 "cilium bpf endpoint list", 245 "cilium bpf ct list global", 246 "cilium bpf nat list", 247 "cilium bpf proxy list", 248 "cilium bpf ipcache list", 249 "cilium bpf policy get --all --numeric", 250 "cilium bpf sha list", 251 "cilium map list --verbose", 252 "cilium status --verbose", 253 "cilium identity list", 254 "cilium-health status", 255 "cilium policy selectors -o json", 256 "cilium node list", 257 } 258 var commands []string 259 260 stateDir := filepath.Join(defaults.RuntimePath, defaults.StateDir) 261 if len(k8sPods) == 0 { // Assuming this is a non k8s deployment 262 dst := filepath.Join(cmdDir, defaults.StateDir) 263 commands = append(commands, fmt.Sprintf("cp -r %s %s", stateDir, dst)) 264 for _, cmd := range ciliumCommands { 265 // Add the host flag if set 266 if len(host) > 0 { 267 cmd = fmt.Sprintf("%s -H %s", cmd, host) 268 } 269 commands = append(commands, cmd) 270 } 271 } else { // Found k8s pods 272 for _, pod := range k8sPods { 273 dst := filepath.Join(cmdDir, fmt.Sprintf("%s-%s", pod, defaults.StateDir)) 274 kubectlArg := fmt.Sprintf("%s/%s:%s", k8sNamespace, pod, stateDir) 275 // kubectl cp kube-system/cilium-xrzwr:/var/run/cilium/state cilium-xrzwr-state 276 commands = append(commands, fmt.Sprintf("kubectl cp %s %s", kubectlArg, dst)) 277 for _, cmd := range ciliumCommands { 278 // Add the host flag if set 279 if len(host) > 0 { 280 cmd = fmt.Sprintf("%s -H %s", cmd, host) 281 } 282 commands = append(commands, podPrefix(pod, cmd)) 283 } 284 } 285 } 286 287 return commands 288 } 289 290 func k8sCommands(allCommands []string, pods []string) []string { 291 // These commands do not require a pod argument 292 var commands = []string{ 293 "kubectl get nodes -o wide", 294 "kubectl describe nodes", 295 "kubectl get pods,svc --all-namespaces", 296 "kubectl version", 297 fmt.Sprintf("kubectl get cm cilium-config -n %s", k8sNamespace), 298 } 299 300 // Prepare to run all the commands inside of the pod(s) 301 for _, pod := range pods { 302 for _, cmd := range allCommands { 303 // Add the host flag if set 304 if strings.HasPrefix(cmd, "cilium") && 305 !strings.Contains(cmd, "-H") && len(host) > 0 { 306 cmd = fmt.Sprintf("%s -H %s", cmd, host) 307 } 308 309 if !strings.Contains(cmd, "kubectl exec") { 310 cmd = podPrefix(pod, cmd) 311 } 312 commands = append(commands, cmd) 313 } 314 315 // Retrieve current version of pod logs 316 cmd := fmt.Sprintf("kubectl -n %s logs --timestamps %s", k8sNamespace, pod) 317 commands = append(commands, cmd) 318 319 // Retrieve previous version of pod logs 320 cmd = fmt.Sprintf("kubectl -n %s logs --timestamps -p %s", k8sNamespace, pod) 321 commands = append(commands, cmd) 322 323 cmd = fmt.Sprintf("kubectl -n %s describe pod %s", k8sNamespace, pod) 324 commands = append(commands, cmd) 325 } 326 327 if len(pods) == 0 { 328 allCommands = append(allCommands, commands...) 329 return allCommands 330 } 331 332 return commands 333 }