From 78b8a1430debfb9361e425e52e8366409b9e5cd2 Mon Sep 17 00:00:00 2001 From: Josh Gao Date: Wed, 9 Nov 2016 01:00:41 -0800 Subject: [PATCH] versioner: use a virtual filesystem for input files. Use an InMemoryFileSystem to store and share input files across compilations. This improves the result of `time versioner` further, from: versioner 109.12s user 17.43s system 2433% cpu 5.201 total to: versioner 112.20s user 1.38s system 2416% cpu 4.700 total Bug: http://b/32748936 Test: python run_tests.py Change-Id: I72d37b7c30850b8399cc40338247700fe3e7b2f9 --- tools/versioner/src/Android.mk | 3 +- tools/versioner/src/Driver.cpp | 23 ++++---- tools/versioner/src/Driver.h | 9 ++- tools/versioner/src/VFS.cpp | 92 +++++++++++++++++++++++++++++++ tools/versioner/src/VFS.h | 27 +++++++++ tools/versioner/src/versioner.cpp | 12 ++-- 6 files changed, 148 insertions(+), 18 deletions(-) create mode 100644 tools/versioner/src/VFS.cpp create mode 100644 tools/versioner/src/VFS.h diff --git a/tools/versioner/src/Android.mk b/tools/versioner/src/Android.mk index 713d6c169..c90e02f04 100644 --- a/tools/versioner/src/Android.mk +++ b/tools/versioner/src/Android.mk @@ -31,7 +31,8 @@ LOCAL_SRC_FILES := \ Driver.cpp \ Preprocessor.cpp \ SymbolDatabase.cpp \ - Utils.cpp + Utils.cpp \ + VFS.cpp LOCAL_SHARED_LIBRARIES := libclang libLLVM libbase diff --git a/tools/versioner/src/Driver.cpp b/tools/versioner/src/Driver.cpp index dad8a1e1c..215dc3c72 100644 --- a/tools/versioner/src/Driver.cpp +++ b/tools/versioner/src/Driver.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -92,7 +93,8 @@ static IntrusiveRefCntPtr constructDiags() { // Run it once to generate flags for each target, and memoize the results. static std::unordered_map> cc1_flags; static const char* filename_placeholder = "__VERSIONER_PLACEHOLDER__"; -static void generateTargetCC1Flags(CompilationType type, +static void generateTargetCC1Flags(llvm::IntrusiveRefCntPtr vfs, + CompilationType type, const std::vector& include_dirs) { std::vector cmd = { "versioner" }; cmd.push_back("-std=c11"); @@ -122,12 +124,8 @@ static void generateTargetCC1Flags(CompilationType type, cmd.push_back("-nostdinc"); if (add_include) { - const char* top = getenv("ANDROID_BUILD_TOP"); - if (!top) { - errx(1, "-i passed, but ANDROID_BUILD_TOP is unset"); - } cmd.push_back("-include"); - cmd.push_back(to_string(top) + "/bionic/libc/include/android/versioning.h"); + cmd.push_back("android/versioning.h"); } for (const auto& dir : include_dirs) { @@ -138,7 +136,7 @@ static void generateTargetCC1Flags(CompilationType type, cmd.push_back(filename_placeholder); auto diags = constructDiags(); - driver::Driver driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags); + driver::Driver driver("versioner", llvm::sys::getDefaultTargetTriple(), *diags, vfs); driver.setCheckInputsExist(false); llvm::SmallVector driver_args; @@ -192,7 +190,8 @@ static std::vector getCC1Command(CompilationType type, const std::s return result; } -void initializeTargetCC1FlagCache(const std::set& types, +void initializeTargetCC1FlagCache(llvm::IntrusiveRefCntPtr vfs, + const std::set& types, const std::unordered_map& reqs) { if (!cc1_flags.empty()) { errx(1, "reinitializing target CC1 flag cache?"); @@ -201,14 +200,14 @@ void initializeTargetCC1FlagCache(const std::set& types, auto start = std::chrono::high_resolution_clock::now(); std::vector threads; for (const CompilationType type : types) { - threads.emplace_back([type, &reqs]() { + threads.emplace_back([type, &vfs, &reqs]() { const auto& arch_req_it = reqs.find(type.arch); if (arch_req_it == reqs.end()) { errx(1, "CompilationRequirement map missing entry for CompilationType %s", to_string(type).c_str()); } - generateTargetCC1Flags(type, arch_req_it->second.dependencies); + generateTargetCC1Flags(vfs, type, arch_req_it->second.dependencies); }); } for (auto& thread : threads) { @@ -226,7 +225,8 @@ void initializeTargetCC1FlagCache(const std::set& types, } } -void compileHeader(HeaderDatabase* header_database, CompilationType type, +void compileHeader(llvm::IntrusiveRefCntPtr vfs, + HeaderDatabase* header_database, CompilationType type, const std::string& filename) { auto diags = constructDiags(); std::vector cc1_flags = getCC1Command(type, filename); @@ -239,6 +239,7 @@ void compileHeader(HeaderDatabase* header_database, CompilationType type, clang::CompilerInstance Compiler; Compiler.setInvocation(invocation.release()); Compiler.setDiagnostics(diags.get()); + Compiler.setVirtualFileSystem(vfs); VersionerASTAction versioner_action(header_database, type); if (!Compiler.ExecuteAction(versioner_action)) { diff --git a/tools/versioner/src/Driver.h b/tools/versioner/src/Driver.h index c1a4b24f3..48683e431 100644 --- a/tools/versioner/src/Driver.h +++ b/tools/versioner/src/Driver.h @@ -20,16 +20,21 @@ #include #include +#include + #include "Arch.h" #include "DeclarationDatabase.h" +#include "VFS.h" struct CompilationRequirements { std::vector headers; std::vector dependencies; }; -void initializeTargetCC1FlagCache(const std::set& types, +void initializeTargetCC1FlagCache(llvm::IntrusiveRefCntPtr vfs, + const std::set& types, const std::unordered_map& reqs); -void compileHeader(HeaderDatabase* header_database, CompilationType type, +void compileHeader(llvm::IntrusiveRefCntPtr vfs, + HeaderDatabase* header_database, CompilationType type, const std::string& filename); diff --git a/tools/versioner/src/VFS.cpp b/tools/versioner/src/VFS.cpp new file mode 100644 index 000000000..cd6d367b2 --- /dev/null +++ b/tools/versioner/src/VFS.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include "Utils.h" + +using android::base::unique_fd; +using namespace clang::vfs; + +static void addDirectoryToVFS(InMemoryFileSystem* vfs, const std::string& path) { + char* paths[] = { const_cast(path.c_str()), nullptr }; + FTS* fts = fts_open(paths, FTS_COMFOLLOW | FTS_LOGICAL | FTS_NOCHDIR, nullptr); + if (!fts) { + err(1, "failed to open directory %s", path.c_str()); + } + + while (FTSENT* ent = fts_read(fts)) { + if ((ent->fts_info & FTS_F) == 0) { + continue; + } + + const char* file_path = ent->fts_accpath; + unique_fd fd(open(file_path, O_RDONLY | O_CLOEXEC)); + if (fd == -1) { + err(1, "failed to open header '%s'", file_path); + } + + auto buffer_opt = llvm::MemoryBuffer::getOpenFileSlice(fd, file_path, -1, 0); + if (!buffer_opt) { + errx(1, "failed to map header '%s'", file_path); + } + + if (!vfs->addFile(file_path, ent->fts_statp->st_mtim.tv_sec, std::move(buffer_opt.get()))) { + errx(1, "failed to add file '%s'", file_path); + } + } + + fts_close(fts); +} + +llvm::IntrusiveRefCntPtr createCommonVFS(const std::string& header_dir, + const std::string& dependency_dir, + bool add_versioning_header) { + auto vfs = std::make_unique(); + addDirectoryToVFS(vfs.get(), header_dir); + if (!dependency_dir.empty()) { + addDirectoryToVFS(vfs.get(), dependency_dir); + } + + if (add_versioning_header) { + const char* top = getenv("ANDROID_BUILD_TOP"); + if (!top) { + errx(1, "-i passed, but ANDROID_BUILD_TOP is unset"); + } + + std::string header_path = std::string(top) + "/bionic/libc/include/android/versioning.h"; + auto buffer_opt = llvm::MemoryBuffer::getFile(header_path); + if (!buffer_opt) { + err(1, "failed to open %s", header_path.c_str()); + } + vfs->addFile("android/versioning.h", 0, std::move(buffer_opt.get())); + } + + return llvm::IntrusiveRefCntPtr(vfs.release()); +} diff --git a/tools/versioner/src/VFS.h b/tools/versioner/src/VFS.h new file mode 100644 index 000000000..e2ab002ee --- /dev/null +++ b/tools/versioner/src/VFS.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include + +llvm::IntrusiveRefCntPtr createCommonVFS(const std::string& header_dir, + const std::string& dependency_dir, + bool add_versioning_header); diff --git a/tools/versioner/src/versioner.cpp b/tools/versioner/src/versioner.cpp index f0a23393e..5aaf47ae1 100644 --- a/tools/versioner/src/versioner.cpp +++ b/tools/versioner/src/versioner.cpp @@ -44,6 +44,8 @@ #include "Preprocessor.h" #include "SymbolDatabase.h" #include "Utils.h" +#include "VFS.h" + #include "versioner.h" using namespace std::string_literals; @@ -138,6 +140,8 @@ static std::unique_ptr compileHeaders(const std::set threads; @@ -151,7 +155,7 @@ static std::unique_ptr compileHeaders(const std::set> jobs; for (CompilationType type : types) { @@ -165,16 +169,16 @@ static std::unique_ptr compileHeaders(const std::set