github.com/lei006/gmqtt-broker@v0.0.1/plugins/bridge/csvlog.go (about) 1 package bridge 2 3 /* 4 Copyright (c) 2021, Gary Barnett @thinkovation. Released under the Apache 2 License 5 6 CSVLog is a bridge plugin for HMQ that implements CSV logging of messages. See CSVLog.md for more information 7 8 */ 9 10 import ( 11 "encoding/csv" 12 "encoding/json" 13 "errors" 14 "fmt" 15 "io/ioutil" 16 "os" 17 "path/filepath" 18 "sort" 19 "strings" 20 "sync" 21 "time" 22 23 "go.uber.org/zap" 24 ) 25 26 type csvBridgeConfig struct { 27 FileName string `json:"fileName"` 28 LogFileMaxSizeMB int64 `json:"logFileMaxSizeMB"` 29 LogFileMaxFiles int64 `json:"logFileMaxFiles"` 30 WriteIntervalSecs int64 `json:"writeIntervalSecs"` 31 CommandTopic string `json:"commandTopic"` 32 Filters []string `json:"filters"` 33 } 34 35 type csvLog struct { 36 config csvBridgeConfig 37 buffer []string 38 msgchan chan (*Elements) 39 40 sync.RWMutex 41 } 42 43 // rotateLog performs a log rotation - copying the current logfile to the base file name plus a timestamp 44 func (c *csvLog) rotateLog(withPrune bool) error { 45 c.Lock() 46 filename := c.config.FileName 47 c.Unlock() 48 49 basename := strings.TrimSuffix(filename, filepath.Ext(filename)) 50 newpath := basename + time.Now().Format("-20060102T150405") + filepath.Ext(filename) 51 renameError := os.Rename(filename, newpath) 52 if renameError != nil { 53 return renameError 54 } 55 outfile, _ := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 56 outfile.Close() 57 // Whenever we rotate a logfile we prune 58 if withPrune { 59 c.logFilePrune() 60 61 } 62 return nil 63 64 } 65 66 // writeToLog takes an array of elements and writes them to the logfile (or to log or stdout) spefified in 67 // the configuration 68 func (c *csvLog) writeToLog(els []Elements) error { 69 70 c.RLock() 71 fname := c.config.FileName 72 c.RUnlock() 73 if fname == "" { 74 fname = "CSVLOG.CSV" 75 } 76 77 if fname == "{LOG}" { 78 for _, value := range els { 79 t := time.Unix(value.Timestamp, 0) 80 log.Info(t.Format("2006-01-02T15:04:05") + " " + value.ClientID + " " + value.Username + " " + value.Action + " " + value.Topic + " " + value.Payload) 81 } 82 return nil 83 } 84 if fname == "{STDOUT}" { 85 for _, value := range els { 86 t := time.Unix(value.Timestamp, 0) 87 fmt.Println(t.Format("2006-01-02T15:04:05") + " " + value.ClientID + " " + value.Username + " " + value.Action + " " + value.Topic + " " + value.Payload) 88 } 89 return nil 90 } 91 92 var mbsize int64 93 fileStat, fileStatErr := os.Stat(fname) 94 if fileStatErr != nil { 95 log.Warn("Could not get CSVLog info. Received Err " + fileStatErr.Error()) 96 mbsize = 0 97 } else { 98 mbsize = fileStat.Size() / 1024 / 1024 99 } 100 if mbsize > c.config.LogFileMaxSizeMB && c.config.LogFileMaxSizeMB != 0 { 101 rotateErr := c.rotateLog(true) 102 if rotateErr != nil { 103 log.Warn("Unable to rotate outputfile") 104 } 105 } 106 outfile, outfileOpenError := os.OpenFile(fname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 107 defer outfile.Close() 108 if outfileOpenError != nil { 109 log.Warn("Could not open CSV Log file to write") 110 return errors.New("Could not write to CSV Log File") 111 } 112 113 writer := csv.NewWriter(outfile) 114 defer writer.Flush() 115 116 for _, value := range els { 117 t := time.Unix(value.Timestamp, 0) 118 var outrow = []string{t.Format("2006-01-02T15:04:05"), value.ClientID, value.Username, value.Action, value.Topic, value.Payload} 119 writeOutRowError := writer.Write(outrow) 120 if writeOutRowError != nil { 121 log.Warn("Could not write msg to CSV Log") 122 } 123 } 124 return nil 125 126 } 127 128 // Worker should be invoked as a goroutine - It listens on the csvlog message channel for incoming messages 129 // for performance we batch messages into an outqueue and write them in bulk when a timer expires 130 func (c *csvLog) Worker() { 131 log.Info("Running CSVLog worker") 132 var outqueue []Elements 133 134 for true { 135 c.RLock() 136 waitInterval := c.config.WriteIntervalSecs 137 c.RUnlock() 138 139 timer := time.NewTimer(time.Second * time.Duration(waitInterval)) 140 141 select { 142 143 case p := <-c.msgchan: 144 c.RLock() 145 146 oktopublish := false 147 148 // Check to see if any filters are defined. If there are none we assume we're logging everything 149 if len(c.config.Filters) != 0 { 150 // We pick up a Read lock here to parse the c.config.Filters string array 151 // as it's a read lock, and write locks will be rare 152 // it feels as if this will be fine. 153 // If there is contention, it _might_ make sense to quickly lock c, get 154 // the filters and release the lock, then process the filters with no locks 155 // but I think it's unlikely 156 157 for _, filt := range c.config.Filters { 158 if topicMatch(p.Topic, filt) { 159 oktopublish = true 160 break 161 162 } 163 164 } 165 166 } else { 167 oktopublish = true 168 } 169 if oktopublish { 170 var el Elements 171 el.Action = p.Action 172 el.ClientID = p.ClientID 173 el.Payload = p.Payload 174 el.Size = p.Size 175 el.Timestamp = p.Timestamp 176 el.Topic = p.Topic 177 el.Username = p.Username 178 outqueue = append(outqueue, el) 179 } 180 c.RUnlock() 181 break 182 case <-timer.C: 183 if len(outqueue) > 0 { 184 writeResult := c.writeToLog(outqueue) 185 if writeResult != nil { 186 log.Warn("Trouble writing to CSV Log") 187 } 188 outqueue = nil 189 190 } 191 break 192 } 193 194 } 195 } 196 197 // LoadCSVLogConfig loads the configuration file - it currently looks in 198 // "./plugins/csvlog/csvlogconfig.json" (following the example of the default location of the kafka plugin config file) 199 // if it doesn't find it there it looks in two further places - the current directory and 200 // an "assets" folder under the current directory (This is for compatibility with a couple of deployed) 201 // implementations. 202 func LoadCSVLogConfig() csvBridgeConfig { 203 // Check to see if the CSVLOGCONFFILE environment variable is set and if so 204 // check that it does actually point to a file 205 csvLogConfigFile := os.Getenv("CSVLOGCONFFILE") 206 if csvLogConfigFile != "" { 207 if _, err := os.Stat(csvLogConfigFile); os.IsNotExist(err) { 208 csvLogConfigFile = "" 209 } 210 } 211 // If csvLogConfigFile is blank look in the plugins directory, 212 // then the current directory for the csvLogConfigFile. If it's still not found we use a default config 213 // If the file does not exist, we use default parameters 214 if csvLogConfigFile == "" { 215 csvLogConfigFile = "./plugins/csvlog/csvlogconfig.json" 216 } 217 if _, err := os.Stat(csvLogConfigFile); os.IsNotExist(err) { 218 219 if _, err := os.Stat("csvlogconfig.json"); os.IsNotExist(err) { 220 csvLogConfigFile = "" 221 } else { 222 csvLogConfigFile = "csvlogconfig.json" 223 } 224 } 225 226 var configUnmarshalErr error 227 var config csvBridgeConfig 228 if csvLogConfigFile != "" { 229 log.Info("Trying to load config file from " + csvLogConfigFile) 230 content, err := ioutil.ReadFile(csvLogConfigFile) 231 if err != nil { 232 log.Info("Read config file error: ", zap.Error(err)) 233 } 234 configUnmarshalErr = json.Unmarshal(content, &config) 235 } 236 237 if configUnmarshalErr != nil || config.FileName == "" { 238 log.Warn("Unable to load csvlog config file, so using default settings") 239 config.FileName = "/var/log/csvlog.log" 240 config.CommandTopic = "CSVLOG/command" 241 config.WriteIntervalSecs = 10 242 config.LogFileMaxSizeMB = 1 243 config.LogFileMaxFiles = 4 244 245 } 246 return config 247 248 } 249 250 // InitCSVLog initialises a CSVLOG plugin 251 // It does this by loading a config file if one can be found. The default filename follows the same 252 // convention as the kafka plugin - ie it's in "./plugins/csvlog/csvlogconfig.json" but an 253 // environment var - CSVLOGCONFFILE - can be set to provide a different location. 254 // 255 // Once the config is set the worker is started 256 func InitCSVLog() *csvLog { 257 log.Info("Trying to init CSVLOG") 258 259 c := &csvLog{config: LoadCSVLogConfig()} 260 c.msgchan = make(chan *Elements, 200) 261 //Start the csvlog worker 262 go c.Worker() 263 return c 264 265 } 266 267 // topicMatch accepts a topic name and a filter string, it then evaluates the 268 // topic against the filter string and returns true if there is a match. 269 // 270 // The CSV bridge can be configured with 0, 1 or more filters - Where there are no 271 // filters specified, every message will be re-published. Where there are filters, any message 272 // that passes any of the filter tests will be re-published. 273 func topicMatch(topic string, filter string) bool { 274 if topic == filter || filter == "#" { 275 return true 276 } 277 topicComponents := strings.Split(topic, "/") 278 filterComponents := strings.Split(filter, "/") 279 currentpos := 0 280 filterComponentsLength := len(filterComponents) 281 currentFilterComponent := "" 282 if filterComponentsLength > 0 { 283 currentFilterComponent = filterComponents[currentpos] 284 } 285 for _, topicVal := range topicComponents { 286 if currentFilterComponent == "" { 287 return false 288 } 289 if currentFilterComponent == "#" { 290 return true 291 } 292 if currentFilterComponent != "+" && currentFilterComponent != topicVal { 293 return false 294 } 295 currentpos++ 296 if filterComponentsLength > currentpos { 297 currentFilterComponent = filterComponents[currentpos] 298 } else { 299 currentFilterComponent = "" 300 } 301 } 302 return true 303 } 304 305 // logFilePrune checks the number of rotated logfiles and prunes them 306 func (c *csvLog) logFilePrune() error { 307 308 // List the rotated files 309 c.RLock() 310 filename := c.config.FileName 311 maxfiles := c.config.LogFileMaxFiles 312 c.RUnlock() 313 if maxfiles == 0 { 314 return nil 315 } 316 317 fileExt := filepath.Ext(filename) 318 fileDir := filepath.Dir(filename) 319 baseFileName := strings.TrimSuffix(filepath.Base(filename), fileExt) 320 321 files, err := ioutil.ReadDir(fileDir) 322 if err != nil { 323 return err 324 } 325 326 var foundFiles []string 327 for _, file := range files { 328 if strings.HasPrefix(file.Name(), baseFileName+"-") { 329 330 foundFiles = append(foundFiles, file.Name()) 331 332 } 333 } 334 if len(foundFiles) >= int(maxfiles) { 335 fmt.Println("Found ", len(foundFiles), " files") 336 sort.Strings(foundFiles) 337 for i := 0; i < len(foundFiles)-int(maxfiles); i++ { 338 fileDeleteError := os.Remove(fileDir + "//" + foundFiles[i]) 339 log.Info("Pruning logfile " + fileDir + "//" + foundFiles[i]) 340 if fileDeleteError != nil { 341 log.Warn("Could not delete file " + fileDir + "//" + foundFiles[i]) 342 343 } 344 345 } 346 347 } 348 349 return nil 350 351 } 352 353 // Publish implements the bridge interface - it accepts an Element then checks to see if that element is a 354 // message published to the admin topic for the plugin 355 // 356 func (c *csvLog) Publish(e *Elements) (bool, error) { 357 // A short-lived lock on c allows us to 358 // get the Command topic then release the lock 359 // This then allows us to process the command - which may 360 // take its a write lock on c (to update values) and then 361 // return here where we'll pick up a 362 // read lock to iterate over the c.config.filters 363 // We're trying to minimise the time spent in this function 364 // and to limit the overall time spent in any write locks. 365 c.RLock() 366 //CSVLOG allows you to configure a CommandTopic which is a topic to which commands affecting the behaviour of CSVLog can be sent 367 //The simplest would be a message with a payload of "RELOAD" which will reload the configuration allowing configuration changes to be 368 //made at runtime without restarting the broker 369 CommandTopic := c.config.CommandTopic 370 OutFile := c.config.FileName 371 c.RUnlock() 372 // If the outfile is set to "{NULL}" we don't do anything with the message - we just return nil 373 // This feature is here to allow CSVLOG to be enabled/disabled at runtime 374 if OutFile == "{NULL}" { 375 return false, nil 376 } 377 378 if e.Topic == CommandTopic { 379 380 log.Info("CSVLOG Command Received") 381 382 // Process Command 383 // These are going to be rare ocurrences, so in this implementation 384 // we will process the command here - but if we _really_ want to 385 // squeeze delays out, we could have a worker sitting on a 386 // command channel processing any commands. 387 if e.Payload == "RELOADCONFIG" { 388 newConfig := LoadCSVLogConfig() 389 c.Lock() 390 c.config = newConfig 391 c.Unlock() 392 393 } 394 if e.Payload == "ROTATEFILE" { 395 396 c.rotateLog(true) 397 398 } 399 if e.Payload == "ROTATEFILENOPRUNE" { 400 401 c.rotateLog(false) 402 403 } 404 // We could return without doing anything more here, but 405 // for now we move ahead with the filter processing on the 406 // basis that unless we either filter for "all" (with #) or 407 // filter for the CommandTopic, they won't be logged - but we 408 // may have a reason for wanting to track commands too 409 } 410 // Push the message into the channel and return 411 // the channel is buffered and is read by a goroutine so this should block for the shortest possible time 412 c.msgchan <- e 413 return false, nil 414 }