github.com/turbot/steampipe@v1.7.0-rc.0.0.20240517123944-7cef272d4458/pkg/steampipeconfig/connection_state_map.go (about) 1 package steampipeconfig 2 3 import ( 4 "encoding/json" 5 "github.com/turbot/steampipe/pkg/error_helpers" 6 "log" 7 "os" 8 "time" 9 10 sdkplugin "github.com/turbot/steampipe-plugin-sdk/v5/plugin" 11 "github.com/turbot/steampipe/pkg/constants" 12 "github.com/turbot/steampipe/pkg/filepaths" 13 "github.com/turbot/steampipe/pkg/steampipeconfig/modconfig" 14 "github.com/turbot/steampipe/pkg/utils" 15 "golang.org/x/exp/maps" 16 ) 17 18 type ConnectionStateSummary map[string]int 19 20 type ConnectionStateMap map[string]*ConnectionState 21 22 // GetRequiredConnectionStateMap populates a map of connection data for all connections in connectionMap 23 func GetRequiredConnectionStateMap(connectionMap map[string]*modconfig.Connection, currentConnectionState ConnectionStateMap) (ConnectionStateMap, map[string][]modconfig.Connection, error_helpers.ErrorAndWarnings) { 24 utils.LogTime("steampipeconfig.GetRequiredConnectionStateMap start") 25 defer utils.LogTime("steampipeconfig.GetRequiredConnectionStateMap end") 26 27 var res = error_helpers.ErrorAndWarnings{} 28 requiredState := ConnectionStateMap{} 29 30 // cache plugin file creation times in a dictionary to avoid reloading the same plugin file multiple times 31 pluginModTimeMap := make(map[string]time.Time) 32 33 // map of missing plugins, keyed by plugin alias, value is list of connections using missing plugin 34 missingPluginMap := make(map[string][]modconfig.Connection) 35 36 utils.LogTime("steampipeconfig.getRequiredConnections config - iteration start") 37 // populate file mod time for each referenced plugin 38 for name, connection := range connectionMap { 39 // if the connection is in error, create an error connection state 40 // this may have been set by the loading code 41 if connection.Error != nil { 42 // add error connection state 43 requiredState[connection.Name] = newErrorConnectionState(connection) 44 // if error is a missing plugin, add to missingPluginMap 45 // this will be used to build missing plugin warnings 46 if connection.Error.Error() == constants.ConnectionErrorPluginNotInstalled { 47 missingPluginMap[connection.PluginAlias] = append(missingPluginMap[connection.PluginAlias], *connection) 48 } else { 49 // otherwise add error to result as warning, so we display it 50 res.AddWarning(connection.Error.Error()) 51 } 52 continue 53 } 54 55 // to get here, PluginPath must be set 56 pluginPath := *connection.PluginPath 57 58 // get the plugin file mod time 59 var pluginModTime time.Time 60 var ok bool 61 if pluginModTime, ok = pluginModTimeMap[pluginPath]; !ok { 62 var err error 63 pluginModTime, err = utils.FileModTime(pluginPath) 64 if err != nil { 65 res.Error = err 66 return nil, nil, res 67 } 68 } 69 pluginModTimeMap[pluginPath] = pluginModTime 70 requiredState[name] = NewConnectionState(connection, pluginModTime) 71 // the comments _will_ eventually be set 72 requiredState[name].CommentsSet = true 73 // if schema import is disabled, set desired state as disabled 74 if connection.ImportSchema == modconfig.ImportSchemaDisabled { 75 requiredState[name].State = constants.ConnectionStateDisabled 76 } 77 // NOTE: if the connection exists in the current state, copy the connection mod time 78 // (this will be updated to 'now' later if we are updating the connection) 79 if currentState, ok := currentConnectionState[name]; ok { 80 requiredState[name].ConnectionModTime = currentState.ConnectionModTime 81 } 82 } 83 84 return requiredState, missingPluginMap, res 85 } 86 87 func newErrorConnectionState(connection *modconfig.Connection) *ConnectionState { 88 res := NewConnectionState(connection, time.Now()) 89 res.SetError(connection.Error.Error()) 90 return res 91 } 92 93 func (m ConnectionStateMap) GetSummary() ConnectionStateSummary { 94 res := make(map[string]int, len(m)) 95 for _, c := range m { 96 res[c.State]++ 97 } 98 return res 99 } 100 101 // Pending returns whether there are any connections in the map which are pending 102 // this indicates that the db has just started and RefreshConnections has not been called yet 103 func (m ConnectionStateMap) Pending() bool { 104 return m.ConnectionsInState(constants.ConnectionStatePending, constants.ConnectionStatePendingIncomplete) 105 } 106 107 // Loaded returns whether loading is complete, i.e. all connections are either ready or error 108 // (optionally, a list of connections may be passed, in which case just these connections are checked) 109 func (m ConnectionStateMap) Loaded(connections ...string) bool { 110 // if no connections were passed, check them all 111 if len(connections) == 0 { 112 connections = maps.Keys(m) 113 } 114 115 for _, connectionName := range connections { 116 connectionState, ok := m[connectionName] 117 if !ok { 118 // ignore if we have no state loaded for this connection name 119 continue 120 } 121 log.Println("[TRACE] Checking state for", connectionName) 122 if !connectionState.Loaded() { 123 return false 124 } 125 } 126 return true 127 } 128 129 // ConnectionsInState returns whether there are any connections one of the given states 130 func (m ConnectionStateMap) ConnectionsInState(states ...string) bool { 131 for _, c := range m { 132 for _, state := range states { 133 if c.State == state { 134 return true 135 } 136 } 137 } 138 return false 139 } 140 141 func (m ConnectionStateMap) Save() error { 142 connFilePath := filepaths.ConnectionStatePath() 143 connFileJSON, err := json.MarshalIndent(m, "", " ") 144 if err != nil { 145 log.Println("[ERROR]", "Error while writing state file", err) 146 return err 147 } 148 return os.WriteFile(connFilePath, connFileJSON, 0644) 149 } 150 151 func (m ConnectionStateMap) Equals(other ConnectionStateMap) bool { 152 if m != nil && other == nil { 153 return false 154 } 155 for k, lVal := range m { 156 rVal, ok := other[k] 157 if !ok || !lVal.Equals(rVal) { 158 return false 159 } 160 } 161 for k := range other { 162 if _, ok := m[k]; !ok { 163 return false 164 } 165 } 166 return true 167 } 168 169 // ConnectionModTime returns the latest connection mod time 170 func (m ConnectionStateMap) ConnectionModTime() time.Time { 171 var res time.Time 172 for _, c := range m { 173 if c.ConnectionModTime.After(res) { 174 res = c.ConnectionModTime 175 } 176 } 177 return res 178 } 179 180 func (m ConnectionStateMap) GetFirstSearchPathConnectionForPlugins(searchPath []string) []string { 181 // build map of the connections which we must wait for: 182 // for static plugins, just the first connection in the search path 183 // for dynamic schemas all schemas in the search paths (as we do not know which schema may provide a given table) 184 requiredSchemasMap := m.getFirstSearchPathConnectionMapForPlugins(searchPath) 185 // convert this into a list 186 var requiredSchemas []string 187 for _, connections := range requiredSchemasMap { 188 requiredSchemas = append(requiredSchemas, connections...) 189 } 190 return requiredSchemas 191 } 192 193 func (m ConnectionStateMap) GetPluginToConnectionMap() map[string][]string { 194 res := make(map[string][]string) 195 for connectionName, connectionState := range m { 196 res[connectionState.Plugin] = append(res[connectionState.Plugin], connectionName) 197 } 198 return res 199 } 200 201 // getFirstSearchPathConnectionMapForPlugins builds map of plugin to the connections which must be loaded to ensure we can resolve unqualified queries 202 // for static plugins, just the first connection in the search path is included 203 // for dynamic schemas all search paths are included 204 func (m ConnectionStateMap) getFirstSearchPathConnectionMapForPlugins(searchPath []string) map[string][]string { 205 res := make(map[string][]string) 206 for _, connectionName := range searchPath { 207 // is this in the connection state map 208 connectionState, ok := m[connectionName] 209 if !ok { 210 continue 211 } 212 // if this connection is disabled, skip it 213 if connectionState.Disabled() { 214 continue 215 } 216 217 // get the plugin 218 plugin := connectionState.Plugin 219 // if this is the first connection for this plugin, or this is a dynamic plugin, add to the result map 220 if len(res[plugin]) == 0 || connectionState.SchemaMode == sdkplugin.SchemaModeDynamic { 221 res[plugin] = append(res[plugin], connectionName) 222 } 223 } 224 return res 225 } 226 227 func (m ConnectionStateMap) SetConnectionsToPendingOrIncomplete() { 228 for _, state := range m { 229 if state.State == constants.ConnectionStateReady { 230 state.State = constants.ConnectionStatePending 231 state.ConnectionModTime = time.Now() 232 } else if state.State != constants.ConnectionStateDisabled { 233 state.State = constants.ConnectionStatePendingIncomplete 234 state.ConnectionModTime = time.Now() 235 } 236 } 237 } 238 239 // PopulateFilename sets the Filename, StartLineNumber and EndLineNumber properties 240 // this is required as these fields were added to the table after release 241 func (m ConnectionStateMap) PopulateFilename() { 242 // get the connection from config 243 connections := GlobalConfig.Connections 244 for name, state := range m { 245 // do we have config for this connection ( 246 if connection := connections[name]; connection != nil { 247 state.setFilename(connection) 248 } 249 } 250 }