kythe.io@v0.0.68-0.20240422202219-7225dbc01741/kythe/cxx/common/indexing/KytheCachingOutput.cc (about)

     1  /*
     2   * Copyright 2014 The Kythe Authors. All rights reserved.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *   http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  #include "kythe/cxx/common/indexing/KytheCachingOutput.h"
    18  
    19  #include <cassert>
    20  #include <cstddef>
    21  #include <cstdio>
    22  #include <string>
    23  
    24  #include "absl/strings/str_cat.h"
    25  #include "absl/strings/str_format.h"
    26  #include "google/protobuf/io/coded_stream.h"
    27  #include "kythe/proto/storage.pb.h"
    28  
    29  namespace kythe {
    30  
    31  std::string FileOutputStream::Stats::ToString() const {
    32    return absl::StrCat(
    33        buffers_merged_, " merged ", buffers_split_, " split ", buffers_retired_,
    34        " retired ", hashes_matched_, " matches ",
    35        (buffers_retired_ ? (total_bytes_ / buffers_retired_) : 0),
    36        " bytes/buffer");
    37  }
    38  
    39  FileOutputStream::~FileOutputStream() {
    40    while (!buffers_.empty()) {
    41      // Shake out any less-than-minimum-sized buffers that remain.
    42      EmitAndReleaseTopBuffer();
    43    }
    44    if (show_stats_) {
    45      absl::FPrintF(stderr, "%s\n", stats_.ToString());
    46      fflush(stderr);
    47    }
    48  }
    49  
    50  void FileOutputStream::EnqueueEntry(const proto::Entry& entry) {
    51    if (cache_ == &default_cache_ || buffers_.empty()) {
    52      {
    53        google::protobuf::io::CodedOutputStream coded_stream(stream_);
    54        coded_stream.WriteVarint32(entry.ByteSizeLong());
    55        entry.SerializeToCodedStream(&coded_stream);
    56      }
    57      if (flush_after_each_entry_) {
    58        stream_->Flush();
    59      }
    60      return;
    61    }
    62  
    63    size_t entry_size = entry.ByteSizeLong();
    64    size_t size_size =
    65        google::protobuf::io::CodedOutputStream::VarintSize32(entry_size);
    66    size_t size_delta = entry_size + size_size;
    67    unsigned char* buffer = buffers_.WriteToTop(size_delta);
    68  
    69    google::protobuf::io::CodedOutputStream::WriteVarint32ToArray(entry_size,
    70                                                                  buffer);
    71    if (!entry.SerializeToArray(&buffer[size_size], size_delta - size_size)) {
    72      assert(0 && "bad proto size calculation");
    73    }
    74    stats_.total_bytes_ += size_delta;
    75  
    76    if (buffers_.top_size() >= max_size_) {
    77      ++stats_.buffers_split_;
    78      EmitAndReleaseTopBuffer();
    79      PushBuffer();
    80    }
    81  }
    82  
    83  void FileOutputStream::EmitAndReleaseTopBuffer() {
    84    HashCache::Hash hash;
    85    buffers_.HashTop(&hash);
    86    if (!cache_->SawHash(hash)) {
    87      buffers_.CopyTopToStream(stream_);
    88      if (flush_after_each_entry_) {
    89        stream_->Flush();
    90      }
    91      cache_->RegisterHash(hash);
    92    } else {
    93      ++stats_.hashes_matched_;
    94    }
    95    buffers_.Pop();
    96    ++stats_.buffers_retired_;
    97  }
    98  
    99  void FileOutputStream::PushBuffer() { buffers_.Push(max_size_); }
   100  
   101  void FileOutputStream::PopBuffer() {
   102    if (buffers_.MergeDownIfTooSmall(min_size_, max_size_)) {
   103      ++stats_.buffers_merged_;
   104    } else {
   105      EmitAndReleaseTopBuffer();
   106    }
   107  }
   108  
   109  }  // namespace kythe