github.com/maruel/nin@v0.0.0-20220112143044-f35891e3ce7e/src/util.cc (about) 1 // Copyright 2011 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #include "util.h" 16 17 #ifdef __CYGWIN__ 18 #include <windows.h> 19 #include <io.h> 20 #elif defined( _WIN32) 21 #include <windows.h> 22 #include <io.h> 23 #include <share.h> 24 #endif 25 26 #include <assert.h> 27 #include <errno.h> 28 #include <fcntl.h> 29 #include <stdarg.h> 30 #include <stdio.h> 31 #include <stdlib.h> 32 #include <string.h> 33 #include <sys/stat.h> 34 #include <sys/types.h> 35 36 #ifndef _WIN32 37 #include <unistd.h> 38 #include <sys/time.h> 39 #endif 40 41 #include <vector> 42 43 #if defined(__APPLE__) || defined(__FreeBSD__) 44 #include <sys/sysctl.h> 45 #elif defined(__SVR4) && defined(__sun) 46 #include <unistd.h> 47 #include <sys/loadavg.h> 48 #elif defined(_AIX) && !defined(__PASE__) 49 #include <libperfstat.h> 50 #elif defined(linux) || defined(__GLIBC__) 51 #include <sys/sysinfo.h> 52 #endif 53 54 #if defined(__FreeBSD__) 55 #include <sys/cpuset.h> 56 #endif 57 58 #include "edit_distance.h" 59 60 using namespace std; 61 62 void Fatal(const char* msg, ...) { 63 va_list ap; 64 fprintf(stderr, "ninja: fatal: "); 65 va_start(ap, msg); 66 vfprintf(stderr, msg, ap); 67 va_end(ap); 68 fprintf(stderr, "\n"); 69 #ifdef _WIN32 70 // On Windows, some tools may inject extra threads. 71 // exit() may block on locks held by those threads, so forcibly exit. 72 fflush(stderr); 73 fflush(stdout); 74 ExitProcess(1); 75 #else 76 exit(1); 77 #endif 78 } 79 80 void Warning(const char* msg, va_list ap) { 81 fprintf(stderr, "ninja: warning: "); 82 vfprintf(stderr, msg, ap); 83 fprintf(stderr, "\n"); 84 } 85 86 void Warning(const char* msg, ...) { 87 va_list ap; 88 va_start(ap, msg); 89 Warning(msg, ap); 90 va_end(ap); 91 } 92 93 void Error(const char* msg, va_list ap) { 94 fprintf(stderr, "ninja: error: "); 95 vfprintf(stderr, msg, ap); 96 fprintf(stderr, "\n"); 97 } 98 99 void Error(const char* msg, ...) { 100 va_list ap; 101 va_start(ap, msg); 102 Error(msg, ap); 103 va_end(ap); 104 } 105 106 void Info(const char* msg, va_list ap) { 107 fprintf(stdout, "ninja: "); 108 vfprintf(stdout, msg, ap); 109 fprintf(stdout, "\n"); 110 } 111 112 void Info(const char* msg, ...) { 113 va_list ap; 114 va_start(ap, msg); 115 Info(msg, ap); 116 va_end(ap); 117 } 118 119 void CanonicalizePath(string* path, uint64_t* slash_bits) { 120 size_t len = path->size(); 121 char* str = 0; 122 if (len > 0) 123 str = &(*path)[0]; 124 CanonicalizePath(str, &len, slash_bits); 125 path->resize(len); 126 } 127 128 static bool IsPathSeparator(char c) { 129 #ifdef _WIN32 130 return c == '/' || c == '\\'; 131 #else 132 return c == '/'; 133 #endif 134 } 135 136 void CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits) { 137 // WARNING: this function is performance-critical; please benchmark 138 // any changes you make to it. 139 if (*len == 0) { 140 return; 141 } 142 143 const int kMaxPathComponents = 60; 144 char* components[kMaxPathComponents]; 145 int component_count = 0; 146 147 char* start = path; 148 char* dst = start; 149 const char* src = start; 150 const char* end = start + *len; 151 152 if (IsPathSeparator(*src)) { 153 #ifdef _WIN32 154 155 // network path starts with // 156 if (*len > 1 && IsPathSeparator(*(src + 1))) { 157 src += 2; 158 dst += 2; 159 } else { 160 ++src; 161 ++dst; 162 } 163 #else 164 ++src; 165 ++dst; 166 #endif 167 } 168 169 while (src < end) { 170 if (*src == '.') { 171 if (src + 1 == end || IsPathSeparator(src[1])) { 172 // '.' component; eliminate. 173 src += 2; 174 continue; 175 } else if (src[1] == '.' && (src + 2 == end || IsPathSeparator(src[2]))) { 176 // '..' component. Back up if possible. 177 if (component_count > 0) { 178 dst = components[component_count - 1]; 179 src += 3; 180 --component_count; 181 } else { 182 *dst++ = *src++; 183 *dst++ = *src++; 184 *dst++ = *src++; 185 } 186 continue; 187 } 188 } 189 190 if (IsPathSeparator(*src)) { 191 src++; 192 continue; 193 } 194 195 if (component_count == kMaxPathComponents) 196 Fatal("path has too many components : %s", path); 197 components[component_count] = dst; 198 ++component_count; 199 200 while (src != end && !IsPathSeparator(*src)) 201 *dst++ = *src++; 202 *dst++ = *src++; // Copy '/' or final \0 character as well. 203 } 204 205 if (dst == start) { 206 *dst++ = '.'; 207 *dst++ = '\0'; 208 } 209 210 *len = dst - start - 1; 211 #ifdef _WIN32 212 uint64_t bits = 0; 213 uint64_t bits_mask = 1; 214 215 for (char* c = start; c < start + *len; ++c) { 216 switch (*c) { 217 case '\\': 218 bits |= bits_mask; 219 *c = '/'; 220 NINJA_FALLTHROUGH; 221 case '/': 222 bits_mask <<= 1; 223 } 224 } 225 226 *slash_bits = bits; 227 #else 228 *slash_bits = 0; 229 #endif 230 } 231 232 static inline bool IsKnownShellSafeCharacter(char ch) { 233 if ('A' <= ch && ch <= 'Z') return true; 234 if ('a' <= ch && ch <= 'z') return true; 235 if ('0' <= ch && ch <= '9') return true; 236 237 switch (ch) { 238 case '_': 239 case '+': 240 case '-': 241 case '.': 242 case '/': 243 return true; 244 default: 245 return false; 246 } 247 } 248 249 static inline bool IsKnownWin32SafeCharacter(char ch) { 250 switch (ch) { 251 case ' ': 252 case '"': 253 return false; 254 default: 255 return true; 256 } 257 } 258 259 static inline bool StringNeedsShellEscaping(const string& input) { 260 for (size_t i = 0; i < input.size(); ++i) { 261 if (!IsKnownShellSafeCharacter(input[i])) return true; 262 } 263 return false; 264 } 265 266 static inline bool StringNeedsWin32Escaping(const string& input) { 267 for (size_t i = 0; i < input.size(); ++i) { 268 if (!IsKnownWin32SafeCharacter(input[i])) return true; 269 } 270 return false; 271 } 272 273 void GetShellEscapedString(const string& input, string* result) { 274 assert(result); 275 276 if (!StringNeedsShellEscaping(input)) { 277 result->append(input); 278 return; 279 } 280 281 const char kQuote = '\''; 282 const char kEscapeSequence[] = "'\\'"; 283 284 result->push_back(kQuote); 285 286 string::const_iterator span_begin = input.begin(); 287 for (string::const_iterator it = input.begin(), end = input.end(); it != end; 288 ++it) { 289 if (*it == kQuote) { 290 result->append(span_begin, it); 291 result->append(kEscapeSequence); 292 span_begin = it; 293 } 294 } 295 result->append(span_begin, input.end()); 296 result->push_back(kQuote); 297 } 298 299 300 void GetWin32EscapedString(const string& input, string* result) { 301 assert(result); 302 if (!StringNeedsWin32Escaping(input)) { 303 result->append(input); 304 return; 305 } 306 307 const char kQuote = '"'; 308 const char kBackslash = '\\'; 309 310 result->push_back(kQuote); 311 size_t consecutive_backslash_count = 0; 312 string::const_iterator span_begin = input.begin(); 313 for (string::const_iterator it = input.begin(), end = input.end(); it != end; 314 ++it) { 315 switch (*it) { 316 case kBackslash: 317 ++consecutive_backslash_count; 318 break; 319 case kQuote: 320 result->append(span_begin, it); 321 result->append(consecutive_backslash_count + 1, kBackslash); 322 span_begin = it; 323 consecutive_backslash_count = 0; 324 break; 325 default: 326 consecutive_backslash_count = 0; 327 break; 328 } 329 } 330 result->append(span_begin, input.end()); 331 result->append(consecutive_backslash_count, kBackslash); 332 result->push_back(kQuote); 333 } 334 335 int ReadFile(const string& path, string* contents, string* err) { 336 #ifdef _WIN32 337 // This makes a ninja run on a set of 1500 manifest files about 4% faster 338 // than using the generic fopen code below. 339 err->clear(); 340 HANDLE f = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, 341 OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); 342 if (f == INVALID_HANDLE_VALUE) { 343 err->assign(GetLastErrorString()); 344 return -ENOENT; 345 } 346 347 for (;;) { 348 DWORD len; 349 char buf[64 << 10]; 350 if (!::ReadFile(f, buf, sizeof(buf), &len, NULL)) { 351 err->assign(GetLastErrorString()); 352 contents->clear(); 353 return -1; 354 } 355 if (len == 0) 356 break; 357 contents->append(buf, len); 358 } 359 ::CloseHandle(f); 360 return 0; 361 #else 362 FILE* f = fopen(path.c_str(), "rb"); 363 if (!f) { 364 err->assign(strerror(errno)); 365 return -errno; 366 } 367 368 struct stat st; 369 if (fstat(fileno(f), &st) < 0) { 370 err->assign(strerror(errno)); 371 fclose(f); 372 return -errno; 373 } 374 375 // +1 is for the resize in ManifestParser::Load 376 contents->reserve(st.st_size + 1); 377 378 char buf[64 << 10]; 379 size_t len; 380 while (!feof(f) && (len = fread(buf, 1, sizeof(buf), f)) > 0) { 381 contents->append(buf, len); 382 } 383 if (ferror(f)) { 384 err->assign(strerror(errno)); // XXX errno? 385 contents->clear(); 386 fclose(f); 387 return -errno; 388 } 389 fclose(f); 390 return 0; 391 #endif 392 } 393 394 void SetCloseOnExec(int fd) { 395 #ifndef _WIN32 396 int flags = fcntl(fd, F_GETFD); 397 if (flags < 0) { 398 perror("fcntl(F_GETFD)"); 399 } else { 400 if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) 401 perror("fcntl(F_SETFD)"); 402 } 403 #else 404 HANDLE hd = (HANDLE) _get_osfhandle(fd); 405 if (! SetHandleInformation(hd, HANDLE_FLAG_INHERIT, 0)) { 406 fprintf(stderr, "SetHandleInformation(): %s", GetLastErrorString().c_str()); 407 } 408 #endif // ! _WIN32 409 } 410 411 412 const char* SpellcheckStringV(const string& text, 413 const vector<const char*>& words) { 414 const bool kAllowReplacements = true; 415 const int kMaxValidEditDistance = 3; 416 417 int min_distance = kMaxValidEditDistance + 1; 418 const char* result = NULL; 419 for (vector<const char*>::const_iterator i = words.begin(); 420 i != words.end(); ++i) { 421 int distance = EditDistance(*i, text, kAllowReplacements, 422 kMaxValidEditDistance); 423 if (distance < min_distance) { 424 min_distance = distance; 425 result = *i; 426 } 427 } 428 return result; 429 } 430 431 const char* SpellcheckString(const char* text, ...) { 432 // Note: This takes a const char* instead of a string& because using 433 // va_start() with a reference parameter is undefined behavior. 434 va_list ap; 435 va_start(ap, text); 436 vector<const char*> words; 437 const char* word; 438 while ((word = va_arg(ap, const char*))) 439 words.push_back(word); 440 va_end(ap); 441 return SpellcheckStringV(text, words); 442 } 443 444 #ifdef _WIN32 445 string GetLastErrorString() { 446 DWORD err = GetLastError(); 447 448 char* msg_buf; 449 FormatMessageA( 450 FORMAT_MESSAGE_ALLOCATE_BUFFER | 451 FORMAT_MESSAGE_FROM_SYSTEM | 452 FORMAT_MESSAGE_IGNORE_INSERTS, 453 NULL, 454 err, 455 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), 456 (char*)&msg_buf, 457 0, 458 NULL); 459 string msg = msg_buf; 460 LocalFree(msg_buf); 461 return msg; 462 } 463 464 void Win32Fatal(const char* function, const char* hint) { 465 if (hint) { 466 Fatal("%s: %s (%s)", function, GetLastErrorString().c_str(), hint); 467 } else { 468 Fatal("%s: %s", function, GetLastErrorString().c_str()); 469 } 470 } 471 #endif 472 473 bool islatinalpha(int c) { 474 // isalpha() is locale-dependent. 475 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); 476 } 477 478 string StripAnsiEscapeCodes(const string& in) { 479 string stripped; 480 stripped.reserve(in.size()); 481 482 for (size_t i = 0; i < in.size(); ++i) { 483 if (in[i] != '\33') { 484 // Not an escape code. 485 stripped.push_back(in[i]); 486 continue; 487 } 488 489 // Only strip CSIs for now. 490 if (i + 1 >= in.size()) break; 491 if (in[i + 1] != '[') continue; // Not a CSI. 492 i += 2; 493 494 // Skip everything up to and including the next [a-zA-Z]. 495 while (i < in.size() && !islatinalpha(in[i])) 496 ++i; 497 } 498 return stripped; 499 } 500 501 int GetProcessorCount() { 502 #ifdef _WIN32 503 #ifndef _WIN64 504 // Need to use GetLogicalProcessorInformationEx to get real core count on 505 // machines with >64 cores. See https://stackoverflow.com/a/31209344/21475 506 DWORD len = 0; 507 if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len) 508 && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 509 std::vector<char> buf(len); 510 int cores = 0; 511 if (GetLogicalProcessorInformationEx(RelationProcessorCore, 512 reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>( 513 buf.data()), &len)) { 514 for (DWORD i = 0; i < len; ) { 515 auto info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>( 516 buf.data() + i); 517 if (info->Relationship == RelationProcessorCore && 518 info->Processor.GroupCount == 1) { 519 for (KAFFINITY core_mask = info->Processor.GroupMask[0].Mask; 520 core_mask; core_mask >>= 1) { 521 cores += (core_mask & 1); 522 } 523 } 524 i += info->Size; 525 } 526 if (cores != 0) { 527 return cores; 528 } 529 } 530 } 531 #endif 532 return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); 533 #else 534 // The number of exposed processors might not represent the actual number of 535 // processors threads can run on. This happens when a CPU set limitation is 536 // active, see https://github.com/ninja-build/ninja/issues/1278 537 #if defined(__FreeBSD__) 538 cpuset_t mask; 539 CPU_ZERO(&mask); 540 if (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(mask), 541 &mask) == 0) { 542 return CPU_COUNT(&mask); 543 } 544 #elif defined(CPU_COUNT) 545 cpu_set_t set; 546 if (sched_getaffinity(getpid(), sizeof(set), &set) == 0) { 547 return CPU_COUNT(&set); 548 } 549 #endif 550 return sysconf(_SC_NPROCESSORS_ONLN); 551 #endif 552 } 553 554 #if defined(_WIN32) || defined(__CYGWIN__) 555 static double CalculateProcessorLoad(uint64_t idle_ticks, uint64_t total_ticks) 556 { 557 static uint64_t previous_idle_ticks = 0; 558 static uint64_t previous_total_ticks = 0; 559 static double previous_load = -0.0; 560 561 uint64_t idle_ticks_since_last_time = idle_ticks - previous_idle_ticks; 562 uint64_t total_ticks_since_last_time = total_ticks - previous_total_ticks; 563 564 bool first_call = (previous_total_ticks == 0); 565 bool ticks_not_updated_since_last_call = (total_ticks_since_last_time == 0); 566 567 double load; 568 if (first_call || ticks_not_updated_since_last_call) { 569 load = previous_load; 570 } else { 571 // Calculate load. 572 double idle_to_total_ratio = 573 ((double)idle_ticks_since_last_time) / total_ticks_since_last_time; 574 double load_since_last_call = 1.0 - idle_to_total_ratio; 575 576 // Filter/smooth result when possible. 577 if(previous_load > 0) { 578 load = 0.9 * previous_load + 0.1 * load_since_last_call; 579 } else { 580 load = load_since_last_call; 581 } 582 } 583 584 previous_load = load; 585 previous_total_ticks = total_ticks; 586 previous_idle_ticks = idle_ticks; 587 588 return load; 589 } 590 591 static uint64_t FileTimeToTickCount(const FILETIME & ft) 592 { 593 uint64_t high = (((uint64_t)(ft.dwHighDateTime)) << 32); 594 uint64_t low = ft.dwLowDateTime; 595 return (high | low); 596 } 597 598 double GetLoadAverage() { 599 FILETIME idle_time, kernel_time, user_time; 600 BOOL get_system_time_succeeded = 601 GetSystemTimes(&idle_time, &kernel_time, &user_time); 602 603 double posix_compatible_load; 604 if (get_system_time_succeeded) { 605 uint64_t idle_ticks = FileTimeToTickCount(idle_time); 606 607 // kernel_time from GetSystemTimes already includes idle_time. 608 uint64_t total_ticks = 609 FileTimeToTickCount(kernel_time) + FileTimeToTickCount(user_time); 610 611 double processor_load = CalculateProcessorLoad(idle_ticks, total_ticks); 612 posix_compatible_load = processor_load * GetProcessorCount(); 613 614 } else { 615 posix_compatible_load = -0.0; 616 } 617 618 return posix_compatible_load; 619 } 620 #elif defined(__PASE__) 621 double GetLoadAverage() { 622 return -0.0f; 623 } 624 #elif defined(_AIX) 625 double GetLoadAverage() { 626 perfstat_cpu_total_t cpu_stats; 627 if (perfstat_cpu_total(NULL, &cpu_stats, sizeof(cpu_stats), 1) < 0) { 628 return -0.0f; 629 } 630 631 // Calculation taken from comment in libperfstats.h 632 return double(cpu_stats.loadavg[0]) / double(1 << SBITS); 633 } 634 #elif defined(__UCLIBC__) || (defined(__BIONIC__) && __ANDROID_API__ < 29) 635 double GetLoadAverage() { 636 struct sysinfo si; 637 if (sysinfo(&si) != 0) 638 return -0.0f; 639 return 1.0 / (1 << SI_LOAD_SHIFT) * si.loads[0]; 640 } 641 #elif defined(__HAIKU__) 642 double GetLoadAverage() { 643 return -0.0f; 644 } 645 #else 646 double GetLoadAverage() { 647 double loadavg[3] = { 0.0f, 0.0f, 0.0f }; 648 if (getloadavg(loadavg, 3) < 0) { 649 // Maybe we should return an error here or the availability of 650 // getloadavg(3) should be checked when ninja is configured. 651 return -0.0f; 652 } 653 return loadavg[0]; 654 } 655 #endif // _WIN32 656 657 string ElideMiddle(const string& str, size_t width) { 658 switch (width) { 659 case 0: return ""; 660 case 1: return "."; 661 case 2: return ".."; 662 case 3: return "..."; 663 } 664 const int kMargin = 3; // Space for "...". 665 string result = str; 666 if (result.size() > width) { 667 size_t elide_size = (width - kMargin) / 2; 668 result = result.substr(0, elide_size) 669 + "..." 670 + result.substr(result.size() - elide_size, elide_size); 671 } 672 return result; 673 } 674 675 bool Truncate(const string& path, size_t size, string* err) { 676 #ifdef _WIN32 677 int fh = _sopen(path.c_str(), _O_RDWR | _O_CREAT, _SH_DENYNO, 678 _S_IREAD | _S_IWRITE); 679 int success = _chsize(fh, size); 680 _close(fh); 681 #else 682 int success = truncate(path.c_str(), size); 683 #endif 684 // Both truncate() and _chsize() return 0 on success and set errno and return 685 // -1 on failure. 686 if (success < 0) { 687 *err = strerror(errno); 688 return false; 689 } 690 return true; 691 }