github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/clients/cmd/logstash/spec/outputs/loki_spec.rb (about)

     1  # encoding: utf-8
     2  require "logstash/devutils/rspec/spec_helper"
     3  require "logstash/outputs/loki"
     4  require "logstash/codecs/plain"
     5  require "logstash/event"
     6  require "net/http"
     7  require 'webmock/rspec'
     8  include Loki
     9  
    10  describe LogStash::Outputs::Loki do
    11  
    12    let (:simple_loki_config) { {'url' => 'http://localhost:3100'} }
    13  
    14    context 'when initializing' do
    15      it "should register" do
    16        loki = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
    17        expect { loki.register }.to_not raise_error
    18      end
    19  
    20      it 'should populate loki config with default or initialized values' do
    21        loki = LogStash::Outputs::Loki.new(simple_loki_config)
    22        expect(loki.url).to eql 'http://localhost:3100'
    23        expect(loki.tenant_id).to eql nil
    24        expect(loki.batch_size).to eql 102400
    25        expect(loki.batch_wait).to eql 1
    26      end
    27    end
    28  
    29    context 'when adding en entry to the batch' do
    30      let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
    31      let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
    32      let (:lbs) {{"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
    33      let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
    34      let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"])}
    35      let (:include_lbs) {{"cluster"=>"us-central1"}.sort.to_h}
    36  
    37      it 'should not add empty line' do
    38        plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
    39        emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [])
    40        expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
    41        expect(plugin.batch).to eql nil
    42      end
    43  
    44      it 'should add entry' do
    45        plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
    46        expect(plugin.batch).to eql nil
    47        expect(plugin.add_entry_to_batch(entry)).to eql true
    48        expect(plugin.add_entry_to_batch(entry)).to eql true
    49        expect(plugin.batch).not_to be_nil
    50        expect(plugin.batch.streams.length).to eq 1
    51        expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 2
    52        expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
    53        expect(plugin.batch.size_bytes).to eq 14
    54      end
    55  
    56      it 'should only allowed labels defined in include_fields' do
    57        plugin = LogStash::Plugin.lookup("output", "loki").new(include_loki_config)
    58        expect(plugin.batch).to eql nil
    59        expect(plugin.add_entry_to_batch(include_entry)).to eql true
    60        expect(plugin.add_entry_to_batch(include_entry)).to eql true
    61        expect(plugin.batch).not_to be_nil
    62        expect(plugin.batch.streams.length).to eq 1
    63        expect(plugin.batch.streams[include_lbs.to_s]['entries'].length).to eq 2
    64        expect(plugin.batch.streams[include_lbs.to_s]['labels']).to eq include_lbs
    65        expect(plugin.batch.size_bytes).to eq 14
    66      end
    67  
    68      it 'should not add if full' do
    69        plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config.merge!({'batch_size'=>10}))
    70        expect(plugin.batch).to eql nil
    71        expect(plugin.add_entry_to_batch(entry)).to eql true # first entry is fine.
    72        expect(plugin.batch).not_to be_nil
    73        expect(plugin.batch.streams.length).to eq 1
    74        expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 1
    75        expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
    76        expect(plugin.batch.size_bytes).to eq 7
    77        expect(plugin.add_entry_to_batch(entry)).to eql false # second entry goes over the limit.
    78        expect(plugin.batch).not_to be_nil
    79        expect(plugin.batch.streams.length).to eq 1
    80        expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 1
    81        expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
    82        expect(plugin.batch.size_bytes).to eq 7
    83      end
    84    end
    85  
    86    context 'batch expiration' do
    87      let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
    88  
    89      it 'should not expire if empty' do
    90        loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
    91        sleep(1)
    92        expect(loki.is_batch_expired).to be false
    93      end
    94      it 'should not expire batch if not old' do
    95        loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
    96        expect(loki.add_entry_to_batch(entry)).to eql true
    97        expect(loki.is_batch_expired).to be false
    98      end
    99      it 'should expire if old' do
   100        loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
   101        expect(loki.add_entry_to_batch(entry)).to eql true
   102        sleep(1)
   103        expect(loki.is_batch_expired).to be true
   104      end
   105    end
   106  
   107    context 'channel' do
   108      let (:event) {LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)})}
   109  
   110      it 'should send entry if batch size reached with no tenant' do
   111        loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5,'batch_size'=>10}))
   112        loki.register
   113        sent = Queue.new
   114        allow(loki).to receive(:send) do |batch|
   115          Thread.new do
   116            sent << batch
   117          end
   118        end
   119        loki.receive(event)
   120        loki.receive(event)
   121        sent.deq
   122        sent.deq
   123        loki.close
   124      end
   125      it 'should send entry while closing' do
   126        loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>10,'batch_size'=>10}))
   127        loki.register
   128        sent = Queue.new
   129        allow(loki).to receive(:send) do | batch|
   130          Thread.new  do
   131            sent << batch
   132          end
   133        end
   134        loki.receive(event)
   135        loki.close
   136        sent.deq
   137      end
   138      it 'should send entry when batch is expiring' do
   139        loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5,'batch_size'=>10}))
   140        loki.register
   141        sent = Queue.new
   142        allow(loki).to receive(:send) do | batch|
   143          Thread.new  do
   144            sent << batch
   145          end
   146        end
   147        loki.receive(event)
   148        sent.deq
   149        sleep(0.01) # Adding a minimal sleep. In few cases @batch=nil might happen after evaluating for nil
   150        expect(loki.batch).to be_nil 
   151        loki.close
   152      end
   153    end
   154  
   155    context 'http requests' do
   156      let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
   157  
   158      it 'should send credentials' do
   159        conf = {
   160            'url'=>'http://localhost:3100/loki/api/v1/push',
   161            'username' => 'foo',
   162            'password' => 'bar',
   163            'tenant_id' => 't'
   164        }
   165        loki = LogStash::Outputs::Loki.new(conf)
   166        loki.register
   167        b = Batch.new(entry)
   168        post = stub_request(:post, "http://localhost:3100/loki/api/v1/push").with(
   169            basic_auth: ['foo', 'bar'],
   170            body: b.to_json,
   171            headers:{
   172                'Content-Type' => 'application/json' ,
   173                'User-Agent' => 'loki-logstash',
   174                'X-Scope-OrgID'=>'t',
   175                'Accept'=>'*/*',
   176                'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
   177            }
   178        )
   179        loki.send(b)
   180        expect(post).to have_been_requested.times(1)
   181      end
   182  
   183      it 'should not send credentials' do
   184        conf = {
   185            'url'=>'http://foo.com/loki/api/v1/push',
   186        }
   187        loki = LogStash::Outputs::Loki.new(conf)
   188        loki.register
   189        b = Batch.new(entry)
   190        post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
   191            body: b.to_json,
   192            headers:{
   193                'Content-Type' => 'application/json' ,
   194                'User-Agent' => 'loki-logstash',
   195                'Accept'=>'*/*',
   196                'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
   197            }
   198        )
   199        loki.send(b)
   200        expect(post).to have_been_requested.times(1)
   201      end
   202      it 'should retry 500' do
   203        conf = {
   204            'url'=>'http://foo.com/loki/api/v1/push',
   205            'retries' => 3,
   206        }
   207        loki = LogStash::Outputs::Loki.new(conf)
   208        loki.register
   209        b = Batch.new(entry)
   210        post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
   211            body: b.to_json,
   212        ).to_return(status: [500, "Internal Server Error"])
   213        loki.send(b)
   214        loki.close
   215        expect(post).to have_been_requested.times(3)
   216      end
   217      it 'should retry 429' do
   218        conf = {
   219            'url'=>'http://foo.com/loki/api/v1/push',
   220            'retries' => 2,
   221        }
   222        loki = LogStash::Outputs::Loki.new(conf)
   223        loki.register
   224        b = Batch.new(entry)
   225        post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
   226            body: b.to_json,
   227            ).to_return(status: [429, "stop spamming"])
   228        loki.send(b)
   229        loki.close
   230        expect(post).to have_been_requested.times(2)
   231      end
   232      it 'should not retry 400' do
   233        conf = {
   234            'url'=>'http://foo.com/loki/api/v1/push',
   235            'retries' => 11,
   236        }
   237        loki = LogStash::Outputs::Loki.new(conf)
   238        loki.register
   239        b = Batch.new(entry)
   240        post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
   241            body: b.to_json,
   242            ).to_return(status: [400, "bad request"])
   243        loki.send(b)
   244        loki.close
   245        expect(post).to have_been_requested.times(1)
   246      end
   247      it 'should retry exception' do
   248        conf = {
   249            'url'=>'http://foo.com/loki/api/v1/push',
   250            'retries' => 11,
   251        }
   252        loki = LogStash::Outputs::Loki.new(conf)
   253        loki.register
   254        b = Batch.new(entry)
   255        post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
   256            body: b.to_json,
   257            ).to_raise("some error").then.to_return(status: [200, "fine !"])
   258        loki.send(b)
   259        loki.close
   260        expect(post).to have_been_requested.times(2)
   261      end
   262    end
   263  end