github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/clients/cmd/fluentd/spec/gems/fluent/plugin/loki_output_spec.rb (about) 1 # frozen_string_literal: true 2 3 require 'spec_helper' 4 require 'time' 5 require 'yajl' 6 require 'fluent/test' 7 require 'fluent/test/driver/output' 8 require 'fluent/test/helpers' 9 10 # prevent Test::Unit's AutoRunner from executing during RSpec's rake task 11 Test::Unit::AutoRunner.need_auto_run = false if defined?(Test::Unit::AutoRunner) 12 13 RSpec.describe Fluent::Plugin::LokiOutput do 14 it 'loads config' do 15 driver = Fluent::Test::Driver::Output.new(described_class) 16 17 driver.configure(<<-CONF) 18 type loki 19 url https://logs-us-west1.grafana.net 20 username userID 21 password API_KEY 22 tenant 1234 23 extra_labels {} 24 line_format key_value 25 drop_single_key true 26 remove_keys a, b 27 insecure_tls true 28 <label> 29 job 30 instance instance 31 </label> 32 CONF 33 34 expect(driver.instance.url).to eq 'https://logs-us-west1.grafana.net' 35 expect(driver.instance.username).to eq 'userID' 36 expect(driver.instance.password).to eq 'API_KEY' 37 expect(driver.instance.tenant).to eq '1234' 38 expect(driver.instance.extra_labels).to eq({}) 39 expect(driver.instance.line_format).to eq :key_value 40 expect(driver.instance.record_accessors.keys).to eq %w[job instance] 41 expect(driver.instance.remove_keys).to eq %w[a b] 42 expect(driver.instance.drop_single_key).to eq true 43 expect(driver.instance.insecure_tls).to eq true 44 end 45 46 it 'converts syslog output to loki output' do 47 config = <<-CONF 48 url https://logs-us-west1.grafana.net 49 CONF 50 driver = Fluent::Test::Driver::Output.new(described_class) 51 driver.configure(config) 52 content = File.readlines('spec/gems/fluent/plugin/data/syslog2') 53 chunk = [Time.at(1_546_270_458), content[0]] 54 payload = driver.instance.generic_to_loki([chunk]) 55 expect(payload[0]['stream'].empty?).to eq true 56 expect(payload[0]['values'].count).to eq 1 57 expect(payload[0]['values'][0][0]).to eq '1546270458000000000' 58 expect(payload[0]['values'][0][1]).to eq content[0] 59 end 60 61 it 'converts syslog output with extra labels to loki output' do 62 config = <<-CONF 63 url https://logs-us-west1.grafana.net 64 extra_labels {"env": "test"} 65 CONF 66 driver = Fluent::Test::Driver::Output.new(described_class) 67 driver.configure(config) 68 content = File.readlines('spec/gems/fluent/plugin/data/syslog2') 69 chunk = [Time.at(1_546_270_458), content[0]] 70 payload = driver.instance.generic_to_loki([chunk]) 71 expect(payload[0]['stream']).to eq('env' => 'test') 72 expect(payload[0]['values'].count).to eq 1 73 expect(payload[0]['values'][0][0]).to eq '1546270458000000000' 74 expect(payload[0]['values'][0][1]).to eq content[0] 75 end 76 77 it 'converts multiple syslog output lines to loki output' do 78 config = <<-CONF 79 url https://logs-us-west1.grafana.net 80 CONF 81 driver = Fluent::Test::Driver::Output.new(described_class) 82 driver.configure(config) 83 content = File.readlines('spec/gems/fluent/plugin/data/syslog2') 84 line1 = [Time.at(1_546_270_458), content[0]] 85 line2 = [Time.at(1_546_270_460), content[1]] 86 payload = driver.instance.generic_to_loki([line1, line2]) 87 expect(payload[0]['stream'].empty?).to eq true 88 expect(payload[0]['values'].count).to eq 2 89 expect(payload[0]['values'][0][0]).to eq '1546270458000000000' 90 expect(payload[0]['values'][0][1]).to eq content[0] 91 expect(payload[0]['values'][1][0]).to eq '1546270460000000000' 92 expect(payload[0]['values'][1][1]).to eq content[1] 93 end 94 95 it 'converts multiple syslog output lines with extra labels to loki output' do 96 config = <<-CONF 97 url https://logs-us-west1.grafana.net 98 extra_labels {"env": "test"} 99 CONF 100 driver = Fluent::Test::Driver::Output.new(described_class) 101 driver.configure(config) 102 content = File.readlines('spec/gems/fluent/plugin/data/syslog2') 103 line1 = [Time.at(1_546_270_458), content[0]] 104 line2 = [Time.at(1_546_270_460), content[1]] 105 payload = driver.instance.generic_to_loki([line1, line2]) 106 expect(payload[0]['stream']).to eq('env' => 'test') 107 expect(payload[0]['values'].count).to eq 2 108 expect(payload[0]['values'][0][0]).to eq '1546270458000000000' 109 expect(payload[0]['values'][0][1]).to eq content[0] 110 expect(payload[0]['values'][1][0]).to eq '1546270460000000000' 111 expect(payload[0]['values'][1][1]).to eq content[1] 112 end 113 114 it 'removed non utf-8 characters from log lines' do 115 config = <<-CONF 116 url https://logs-us-west1.grafana.net 117 CONF 118 driver = Fluent::Test::Driver::Output.new(described_class) 119 driver.configure(config) 120 content = File.readlines('spec/gems/fluent/plugin/data/non_utf8.log')[0] 121 chunk = [Time.at(1_546_270_458), { 'message' => content, 'number': 1.2345, 'stream' => 'stdout' }] 122 payload = driver.instance.generic_to_loki([chunk]) 123 expect(payload[0]['stream'].empty?).to eq true 124 expect(payload[0]['values'].count).to eq 1 125 expect(payload[0]['values'][0][0]).to eq '1546270458000000000' 126 expect(payload[0]['values'][0][1]).to eq 'message="? rest of line" number=1.2345 stream=stdout' 127 end 128 129 it 'handle non utf-8 characters from log lines in json format' do 130 config = <<-CONF 131 url https://logs-us-west1.grafana.net 132 line_format json 133 CONF 134 driver = Fluent::Test::Driver::Output.new(described_class) 135 driver.configure(config) 136 content = File.readlines('spec/gems/fluent/plugin/data/non_utf8.log')[0] 137 chunk = [Time.at(1_546_270_458), { 'message' => content, 'number': 1.2345, 'stream' => 'stdout' }] 138 payload = driver.instance.generic_to_loki([chunk]) 139 expect(payload[0]['stream'].empty?).to eq true 140 expect(payload[0]['values'].count).to eq 1 141 expect(payload[0]['values'][0][0]).to eq '1546270458000000000' 142 expect(payload[0]['values'][0][1]).to eq( 143 "{\"message\":\"\xC1 rest of line\",\"number\":1.2345,\"stream\":\"stdout\"}" 144 ) 145 end 146 147 it 'formats record hash as key_value' do 148 config = <<-CONF 149 url https://logs-us-west1.grafana.net 150 CONF 151 driver = Fluent::Test::Driver::Output.new(described_class) 152 driver.configure(config) 153 content = File.readlines('spec/gems/fluent/plugin/data/syslog') 154 line1 = [Time.at(1_546_270_458), { 'message' => content[0], 'stream' => 'stdout' }] 155 payload = driver.instance.generic_to_loki([line1]) 156 body = { 'streams': payload } 157 expect(body[:streams][0]['stream'].empty?).to eq true 158 expect(body[:streams][0]['values'].count).to eq 1 159 expect(body[:streams][0]['values'][0][0]).to eq '1546270458000000000' 160 expect(body[:streams][0]['values'][0][1]).to eq "message=\"#{content[0]}\" stream=stdout" 161 end 162 163 it 'formats record hash as json' do 164 config = <<-CONF 165 url https://logs-us-west1.grafana.net 166 line_format json 167 CONF 168 driver = Fluent::Test::Driver::Output.new(described_class) 169 driver.configure(config) 170 content = File.readlines('spec/gems/fluent/plugin/data/syslog') 171 line1 = [Time.at(1_546_270_458), { 'message' => content[0], 'stream' => 'stdout' }] 172 payload = driver.instance.generic_to_loki([line1]) 173 body = { 'streams': payload } 174 expect(body[:streams][0]['stream'].empty?).to eq true 175 expect(body[:streams][0]['values'].count).to eq 1 176 expect(body[:streams][0]['values'][0][0]).to eq '1546270458000000000' 177 expect(body[:streams][0]['values'][0][1]).to eq Yajl.dump(line1[1]) 178 end 179 180 it 'extracts record key as label' do 181 config = <<-CONF 182 url https://logs-us-west1.grafana.net 183 line_format json 184 <label> 185 stream 186 </label> 187 CONF 188 driver = Fluent::Test::Driver::Output.new(described_class) 189 driver.configure(config) 190 content = File.readlines('spec/gems/fluent/plugin/data/syslog') 191 line1 = [Time.at(1_546_270_458), { 'message' => content[0], 'stream' => 'stdout' }] 192 payload = driver.instance.generic_to_loki([line1]) 193 body = { 'streams': payload } 194 expect(body[:streams][0]['stream']).to eq('stream' => 'stdout') 195 expect(body[:streams][0]['values'].count).to eq 1 196 expect(body[:streams][0]['values'][0][0]).to eq '1546270458000000000' 197 expect(body[:streams][0]['values'][0][1]).to eq Yajl.dump('message' => content[0]) 198 end 199 200 it 'extracts nested record key as label' do 201 config = <<-CONF 202 url https://logs-us-west1.grafana.net 203 line_format json 204 <label> 205 pod $.kubernetes.pod 206 </label> 207 CONF 208 driver = Fluent::Test::Driver::Output.new(described_class) 209 driver.configure(config) 210 content = File.readlines('spec/gems/fluent/plugin/data/syslog') 211 line1 = [Time.at(1_546_270_458), { 'message' => content[0], 'kubernetes' => { 'pod' => 'podname' } }] 212 payload = driver.instance.generic_to_loki([line1]) 213 body = { 'streams': payload } 214 expect(body[:streams][0]['stream']).to eq('pod' => 'podname') 215 expect(body[:streams][0]['values'].count).to eq 1 216 expect(body[:streams][0]['values'][0][0]).to eq '1546270458000000000' 217 expect(body[:streams][0]['values'][0][1]).to eq Yajl.dump('message' => content[0], 'kubernetes' => {}) 218 end 219 220 it 'extracts nested record key as label and drop key after' do 221 config = <<-CONF 222 url https://logs-us-west1.grafana.net 223 line_format json 224 remove_keys kubernetes 225 <label> 226 pod $.kubernetes.pod 227 </label> 228 CONF 229 driver = Fluent::Test::Driver::Output.new(described_class) 230 driver.configure(config) 231 content = File.readlines('spec/gems/fluent/plugin/data/syslog') 232 line1 = [Time.at(1_546_270_458), { 'message' => content[0], 'kubernetes' => { 'pod' => 'podname' } }] 233 payload = driver.instance.generic_to_loki([line1]) 234 body = { 'streams': payload } 235 expect(body[:streams][0]['stream']).to eq('pod' => 'podname') 236 expect(body[:streams][0]['values'].count).to eq 1 237 expect(body[:streams][0]['values'][0][0]).to eq '1546270458000000000' 238 expect(body[:streams][0]['values'][0][1]).to eq Yajl.dump('message' => content[0]) 239 end 240 241 it 'formats as simple string when only 1 record key' do 242 config = <<-CONF 243 url https://logs-us-west1.grafana.net 244 line_format json 245 drop_single_key true 246 <label> 247 stream 248 </label> 249 CONF 250 driver = Fluent::Test::Driver::Output.new(described_class) 251 driver.configure(config) 252 content = File.readlines('spec/gems/fluent/plugin/data/syslog') 253 line1 = [Time.at(1_546_270_458), { 'message' => content[0], 'stream' => 'stdout' }] 254 payload = driver.instance.generic_to_loki([line1]) 255 body = { 'streams': payload } 256 expect(body[:streams][0]['stream']).to eq('stream' => 'stdout') 257 expect(body[:streams][0]['values'].count).to eq 1 258 expect(body[:streams][0]['values'][0][0]).to eq '1546270458000000000' 259 expect(body[:streams][0]['values'][0][1]).to eq content[0] 260 end 261 262 it 'order by timestamp then index when received unordered' do 263 config = <<-CONF 264 url https://logs-us-west1.grafana.net 265 drop_single_key true 266 <label> 267 stream 268 </label> 269 CONF 270 driver = Fluent::Test::Driver::Output.new(described_class) 271 driver.configure(config) 272 lines = [ 273 [Time.at(1_546_270_460), { 'message' => '4', 'stream' => 'stdout' }], 274 [Time.at(1_546_270_459), { 'message' => '2', 'stream' => 'stdout' }], 275 [Time.at(1_546_270_458), { 'message' => '1', 'stream' => 'stdout' }], 276 [Time.at(1_546_270_459), { 'message' => '3', 'stream' => 'stdout' }], 277 [Time.at(1_546_270_450), { 'message' => '0', 'stream' => 'stdout' }], 278 [Time.at(1_546_270_460), { 'message' => '5', 'stream' => 'stdout' }] 279 ] 280 res = driver.instance.generic_to_loki(lines) 281 expect(res[0]['stream']).to eq('stream' => 'stdout') 282 6.times { |i| expect(res[0]['values'][i][1]).to eq i.to_s } 283 end 284 285 it 'raises an LogPostError when http request is not successful' do 286 config = <<-CONF 287 url https://logs-us-west1.grafana.net 288 CONF 289 driver = Fluent::Test::Driver::Output.new(described_class) 290 driver.configure(config) 291 lines = [[Time.at(1_546_270_458), { 'message' => 'foobar', 'stream' => 'stdout' }]] 292 293 # 200 294 success = Net::HTTPSuccess.new(1.0, 200, 'OK') 295 allow(driver.instance).to receive(:loki_http_request) { success } 296 allow(success).to receive(:body).and_return('fake body') 297 expect { driver.instance.write(lines) }.not_to raise_error 298 299 # 205 300 success = Net::HTTPSuccess.new(1.0, 205, 'OK') 301 allow(driver.instance).to receive(:loki_http_request) { success } 302 allow(success).to receive(:body).and_return('fake body') 303 expect { driver.instance.write(lines) }.not_to raise_error 304 305 # 429 306 too_many_requests = Net::HTTPTooManyRequests.new(1.0, 429, 'OK') 307 allow(driver.instance).to receive(:loki_http_request) { too_many_requests } 308 allow(too_many_requests).to receive(:body).and_return('fake body') 309 expect { driver.instance.write(lines) }.to raise_error(described_class::LogPostError) 310 311 # 505 312 server_error = Net::HTTPServerError.new(1.0, 505, 'OK') 313 allow(driver.instance).to receive(:loki_http_request) { server_error } 314 allow(server_error).to receive(:body).and_return('fake body') 315 expect { driver.instance.write(lines) }.to raise_error(described_class::LogPostError) 316 end 317 318 context 'when output is multi-thread' do 319 let(:thread) do 320 class_double( 321 'Thread', 322 current: { _fluentd_plugin_helper_thread_title: 'thread1' } 323 ).as_stubbed_const 324 end 325 326 before do 327 allow(Thread).to receive(:new).and_yield(thread) 328 end 329 330 it 'adds the fluentd_label by default' do 331 config = <<-CONF 332 url https://logs-us-west1.grafana.net 333 334 <buffer> 335 @type memory 336 flush_thread_count 2 337 </buffer> 338 CONF 339 driver = Fluent::Test::Driver::Output.new(described_class) 340 driver.configure(config) 341 content = File.readlines('spec/gems/fluent/plugin/data/syslog2') 342 chunk = [Time.at(1_546_270_458), content[0]] 343 payload = driver.instance.generic_to_loki([chunk]) 344 expect(payload[0]['stream']).to eq('fluentd_thread' => 'thread1') 345 end 346 347 it 'does not add the fluentd_label when configured' do 348 config = <<-CONF 349 url https://logs-us-west1.grafana.net 350 include_thread_label false 351 352 <buffer> 353 @type memory 354 flush_thread_count 2 355 </buffer> 356 CONF 357 driver = Fluent::Test::Driver::Output.new(described_class) 358 driver.configure(config) 359 content = File.readlines('spec/gems/fluent/plugin/data/syslog2') 360 chunk = [Time.at(1_546_270_458), content[0]] 361 payload = driver.instance.generic_to_loki([chunk]) 362 expect(payload[0]['stream'].empty?).to eq(true) 363 end 364 end 365 end