clang  15.0.0git
IncrementalParser.cpp
Go to the documentation of this file.
1 //===--------- IncrementalParser.cpp - Incremental Compilation -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class which performs incremental code compilation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "IncrementalParser.h"
14 
22 #include "clang/Parse/Parser.h"
23 #include "clang/Sema/Sema.h"
24 
25 #include "llvm/Option/ArgList.h"
26 #include "llvm/Support/CrashRecoveryContext.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/Support/Timer.h"
29 
30 #include <sstream>
31 
32 namespace clang {
33 
34 /// A custom action enabling the incremental processing functionality.
35 ///
36 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
37 /// sees a call to \p EndSourceFile it deletes some of the important objects
38 /// such as \p Preprocessor and \p Sema assuming no further input will come.
39 ///
40 /// \p IncrementalAction ensures it keep its underlying action's objects alive
41 /// as long as the \p IncrementalParser needs them.
42 ///
44 private:
45  bool IsTerminating = false;
46 
47 public:
48  IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
49  llvm::Error &Err)
50  : WrapperFrontendAction([&]() {
51  llvm::ErrorAsOutParameter EAO(&Err);
52  std::unique_ptr<FrontendAction> Act;
53  switch (CI.getFrontendOpts().ProgramAction) {
54  default:
55  Err = llvm::createStringError(
56  std::errc::state_not_recoverable,
57  "Driver initialization failed. "
58  "Incremental mode for action %d is not supported",
60  return Act;
61  case frontend::ASTDump:
62  LLVM_FALLTHROUGH;
63  case frontend::ASTPrint:
64  LLVM_FALLTHROUGH;
66  Act = CreateFrontendAction(CI);
67  break;
69  LLVM_FALLTHROUGH;
71  LLVM_FALLTHROUGH;
72  case frontend::EmitObj:
73  LLVM_FALLTHROUGH;
75  Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
76  break;
77  }
78  return Act;
79  }()) {}
80  FrontendAction *getWrapped() const { return WrappedAction.get(); }
82  return TU_Incremental;
83  }
84  void ExecuteAction() override {
86  assert(CI.hasPreprocessor() && "No PP!");
87 
88  // FIXME: Move the truncation aspect of this into Sema, we delayed this till
89  // here so the source manager would be initialized.
93 
94  // Use a code completion consumer?
95  CodeCompleteConsumer *CompletionConsumer = nullptr;
97  CompletionConsumer = &CI.getCodeCompletionConsumer();
98 
99  Preprocessor &PP = CI.getPreprocessor();
101  PP.EnterMainSourceFile();
102 
103  if (!CI.hasSema())
104  CI.createSema(getTranslationUnitKind(), CompletionConsumer);
105  }
106 
107  // Do not terminate after processing the input. This allows us to keep various
108  // clang objects alive and to incrementally grow the current TU.
109  void EndSourceFile() override {
110  // The WrappedAction can be nullptr if we issued an error in the ctor.
111  if (IsTerminating && getWrapped())
113  }
114 
115  void FinalizeAction() {
116  assert(!IsTerminating && "Already finalized!");
117  IsTerminating = true;
118  EndSourceFile();
119  }
120 };
121 
122 IncrementalParser::IncrementalParser(std::unique_ptr<CompilerInstance> Instance,
123  llvm::LLVMContext &LLVMCtx,
124  llvm::Error &Err)
125  : CI(std::move(Instance)) {
126  llvm::ErrorAsOutParameter EAO(&Err);
127  Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
128  if (Err)
129  return;
130  CI->ExecuteAction(*Act);
131  Consumer = &CI->getASTConsumer();
132  P.reset(
133  new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
134  P->Initialize();
135 }
136 
137 IncrementalParser::~IncrementalParser() { Act->FinalizeAction(); }
138 
140 IncrementalParser::ParseOrWrapTopLevelDecl() {
141  // Recover resources if we crash before exiting this method.
142  Sema &S = CI->getSema();
143  llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
144  Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
145  Sema::LocalEagerInstantiationScope LocalInstantiations(S);
146 
147  PTUs.emplace_back(PartialTranslationUnit());
148  PartialTranslationUnit &LastPTU = PTUs.back();
149  // Add a new PTU.
150  ASTContext &C = S.getASTContext();
151  C.addTranslationUnitDecl();
152  LastPTU.TUPart = C.getTranslationUnitDecl();
153 
154  // Skip previous eof due to last incremental input.
155  if (P->getCurToken().is(tok::eof)) {
156  P->ConsumeToken();
157  // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
158  // might want to do that around HandleEndOfTranslationUnit.
159  P->ExitScope();
160  S.CurContext = nullptr;
161  // Start a new PTU.
162  P->EnterScope(Scope::DeclScope);
163  S.ActOnTranslationUnitScope(P->getCurScope());
164  }
165 
167  Sema::ModuleImportState ImportState;
168  for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
169  AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
170  // If we got a null return and something *was* parsed, ignore it. This
171  // is due to a top-level semicolon, an action override, or a parse error
172  // skipping something.
173  if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
174  return llvm::make_error<llvm::StringError>("Parsing failed. "
175  "The consumer rejected a decl",
176  std::error_code());
177  }
178 
179  DiagnosticsEngine &Diags = getCI()->getDiagnostics();
180  if (Diags.hasErrorOccurred()) {
181  TranslationUnitDecl *MostRecentTU = C.getTranslationUnitDecl();
182  TranslationUnitDecl *PreviousTU = MostRecentTU->getPreviousDecl();
183  assert(PreviousTU && "Must have a TU from the ASTContext initialization!");
184  TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
185  assert(FirstTU);
186  FirstTU->RedeclLink.setLatest(PreviousTU);
187  C.TUDecl = PreviousTU;
188  S.TUScope->setEntity(PreviousTU);
189 
190  // Clean up the lookup table
191  if (StoredDeclsMap *Map = PreviousTU->getLookupPtr()) {
192  for (auto I = Map->begin(); I != Map->end(); ++I) {
193  StoredDeclsList &List = I->second;
194  DeclContextLookupResult R = List.getLookupResult();
195  for (NamedDecl *D : R)
196  if (D->getTranslationUnitDecl() == MostRecentTU)
197  List.remove(D);
198  if (List.isNull())
199  Map->erase(I);
200  }
201  }
202 
203  // FIXME: Do not reset the pragma handlers.
204  Diags.Reset();
205  return llvm::make_error<llvm::StringError>("Parsing failed.",
206  std::error_code());
207  }
208 
209  // Process any TopLevelDecls generated by #pragma weak.
210  for (Decl *D : S.WeakTopLevelDecls()) {
211  DeclGroupRef DGR(D);
212  Consumer->HandleTopLevelDecl(DGR);
213  }
214 
215  LocalInstantiations.perform();
216  GlobalInstantiations.perform();
217 
218  Consumer->HandleTranslationUnit(C);
219 
220  return LastPTU;
221 }
222 
224  IncrementalAction *IncrAct = static_cast<IncrementalAction *>(Act);
225  FrontendAction *WrappedAct = IncrAct->getWrapped();
226  if (!WrappedAct->hasIRSupport())
227  return nullptr;
228  return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
229 }
230 
232 IncrementalParser::Parse(llvm::StringRef input) {
233  Preprocessor &PP = CI->getPreprocessor();
234  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
235 
236  std::ostringstream SourceName;
237  SourceName << "input_line_" << InputCount++;
238 
239  // Create an uninitialized memory buffer, copy code in and append "\n"
240  size_t InputSize = input.size(); // don't include trailing 0
241  // MemBuffer size should *not* include terminating zero
242  std::unique_ptr<llvm::MemoryBuffer> MB(
243  llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
244  SourceName.str()));
245  char *MBStart = const_cast<char *>(MB->getBufferStart());
246  memcpy(MBStart, input.data(), InputSize);
247  MBStart[InputSize] = '\n';
248 
249  SourceManager &SM = CI->getSourceManager();
250 
251  // FIXME: Create SourceLocation, which will allow clang to order the overload
252  // candidates for example
253  SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
254 
255  // Create FileID for the current buffer.
256  FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
257  /*LoadedOffset=*/0, NewLoc);
258 
259  // NewLoc only used for diags.
260  if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
261  return llvm::make_error<llvm::StringError>("Parsing failed. "
262  "Cannot enter source file.",
263  std::error_code());
264 
265  auto PTU = ParseOrWrapTopLevelDecl();
266  if (!PTU)
267  return PTU.takeError();
268 
269  if (PP.getLangOpts().DelayedTemplateParsing) {
270  // Microsoft-specific:
271  // Late parsed templates can leave unswallowed "macro"-like tokens.
272  // They will seriously confuse the Parser when entering the next
273  // source file. So lex until we are EOF.
274  Token Tok;
275  do {
276  PP.Lex(Tok);
277  } while (Tok.isNot(tok::eof));
278  }
279 
280  Token AssertTok;
281  PP.Lex(AssertTok);
282  assert(AssertTok.is(tok::eof) &&
283  "Lexer must be EOF when starting incremental parse!");
284 
285  if (CodeGenerator *CG = getCodeGen(Act.get())) {
286  std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
287  CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
288  M->getContext());
289 
290  PTU->TheModule = std::move(M);
291  }
292 
293  return PTU;
294 }
295 
297  CodeGenerator *CG = getCodeGen(Act.get());
298  assert(CG);
299  return CG->GetMangledName(GD);
300 }
301 
302 } // end namespace clang
clang::IncrementalAction::IncrementalAction
IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx, llvm::Error &Err)
Definition: IncrementalParser.cpp:48
clang::CompilerInstance::hasSema
bool hasSema() const
Definition: CompilerInstance.h:501
clang::Preprocessor::enableIncrementalProcessing
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
Definition: Preprocessor.h:1691
clang::CodeGenerator
The primary public interface to the Clang code generator.
Definition: ModuleBuilder.h:43
clang::frontend::EmitAssembly
@ EmitAssembly
Emit a .s file.
Definition: FrontendOptions.h:58
Utils.h
clang::Preprocessor::isIncrementalProcessingEnabled
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
Definition: Preprocessor.h:1688
clang::FrontendOptions::ProgramAction
frontend::ActionKind ProgramAction
The frontend action to perform.
Definition: FrontendOptions.h:445
clang::Preprocessor::Lex
void Lex(Token &Result)
Lex the next token for this preprocessor.
Definition: Preprocessor.cpp:892
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::WrapperFrontendAction::hasCodeCompletionSupport
bool hasCodeCompletionSupport() const override
Does this action support use with code completion?
Definition: FrontendAction.cpp:1196
clang::FrontendAction::hasIRSupport
virtual bool hasIRSupport() const
Does this action support use with IR files?
Definition: FrontendAction.h:197
clang::IncrementalAction::getTranslationUnitKind
TranslationUnitKind getTranslationUnitKind() override
For AST-based actions, the kind of translation unit we're handling.
Definition: IncrementalParser.cpp:81
AttributeLangSupport::C
@ C
Definition: SemaDeclAttr.cpp:55
clang::CreateFrontendAction
std::unique_ptr< FrontendAction > CreateFrontendAction(CompilerInstance &CI)
Construct the FrontendAction of a compiler invocation based on the options specified for the compiler...
Definition: ExecuteCompilerInvocation.cpp:142
clang::IncrementalAction::ExecuteAction
void ExecuteAction() override
Callback to run the program action, using the initialized compiler instance.
Definition: IncrementalParser.cpp:84
memcpy
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
Definition: __clang_cuda_device_functions.h:1549
clang::Parser
Parser - This implements a parser for the C family of languages.
Definition: Parser.h:60
ModuleBuilder.h
clang::frontend::ASTDump
@ ASTDump
Parse ASTs and dump them.
Definition: FrontendOptions.h:40
llvm::Expected
Definition: LLVM.h:41
FrontendAction.h
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
clang::frontend::PluginAction
@ PluginAction
Run a plugin action,.
Definition: FrontendOptions.h:115
clang::CompilerInstance::createCodeCompletionConsumer
void createCodeCompletionConsumer()
Create a code completion consumer using the invocation; note that this will cause the source manager ...
Definition: CompilerInstance.cpp:710
CodeGenAction.h
clang::IncrementalParser::IncrementalParser
IncrementalParser(std::unique_ptr< CompilerInstance > Instance, llvm::LLVMContext &LLVMCtx, llvm::Error &Err)
Definition: IncrementalParser.cpp:122
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:627
clang::ParsedSourceLocation::FileName
std::string FileName
Definition: CommandLineSourceLoc.h:25
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:994
clang::GlobalDecl
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:56
clang::CodeGenerator::GetMangledName
llvm::StringRef GetMangledName(GlobalDecl GD)
Given a global declaration, return a mangled name for this declaration which has been added to this c...
Definition: ModuleBuilder.cpp:337
clang::frontend::ASTPrint
@ ASTPrint
Parse ASTs and print them.
Definition: FrontendOptions.h:43
BackendUtil.h
clang::CodeGenAction
Definition: CodeGenAction.h:24
clang::Preprocessor::EnterSourceFile
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Definition: PPLexerChange.cpp:68
clang::frontend::EmitObj
@ EmitObj
Emit a .o file.
Definition: FrontendOptions.h:76
clang::IncrementalParser::GetMangledName
llvm::StringRef GetMangledName(GlobalDecl GD) const
Uses the CodeGenModule mangled name cache and avoids recomputing.
Definition: IncrementalParser.cpp:296
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:208
clang::TU_Incremental
@ TU_Incremental
The translation unit is a is a complete translation unit that we might incrementally extend later.
Definition: LangOptions.h:829
clang::frontend::ParseSyntaxOnly
@ ParseSyntaxOnly
Parse and perform semantic analysis.
Definition: FrontendOptions.h:112
clang::IncrementalParser::Parse
llvm::Expected< PartialTranslationUnit & > Parse(llvm::StringRef Input)
Parses incremental input by creating an in-memory file.
Definition: IncrementalParser.cpp:232
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:97
clang::IncrementalAction::FinalizeAction
void FinalizeAction()
Definition: IncrementalParser.cpp:115
clang::IncrementalAction::EndSourceFile
void EndSourceFile() override
Perform any per-file post processing, deallocate per-file objects, and run statistics and output file...
Definition: IncrementalParser.cpp:109
clang::FrontendAction::getCompilerInstance
CompilerInstance & getCompilerInstance() const
Definition: FrontendAction.h:119
clang::CompilerInstance::getPreprocessor
Preprocessor & getPreprocessor() const
Return the current preprocessor.
Definition: CompilerInstance.h:442
clang::SrcMgr::C_User
@ C_User
Definition: SourceManager.h:80
clang::Preprocessor::EnterMainSourceFile
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
Definition: Preprocessor.cpp:535
clang::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
clang::CompilerInstance
CompilerInstance - Helper class for managing a single instance of the Clang compiler.
Definition: CompilerInstance.h:72
Sema.h
clang::ASTConsumer::HandleTopLevelDecl
virtual bool HandleTopLevelDecl(DeclGroupRef D)
HandleTopLevelDecl - Handle the specified top-level declaration.
Definition: ASTConsumer.cpp:18
clang::Sema::ModuleImportState
ModuleImportState
An enumeration to represent the transition of states in parsing module fragments and imports.
Definition: Sema.h:3001
clang::CompilerInstance::hasPreprocessor
bool hasPreprocessor() const
Definition: CompilerInstance.h:439
clang::IncrementalAction
A custom action enabling the incremental processing functionality.
Definition: IncrementalParser.cpp:43
clang::getCodeGen
static CodeGenerator * getCodeGen(FrontendAction *Act)
Definition: IncrementalParser.cpp:223
clang::CompilerInstance::hasCodeCompletionConsumer
bool hasCodeCompletionConsumer() const
Definition: CompilerInstance.h:558
IncrementalParser.h
clang::IncrementalAction::getWrapped
FrontendAction * getWrapped() const
Definition: IncrementalParser.cpp:80
clang::Parser::DeclGroupPtrTy
OpaquePtr< DeclGroupRef > DeclGroupPtrTy
Definition: Parser.h:458
clang::Sema
Sema - This implements semantic analysis and AST building for C.
Definition: Sema.h:354
clang::CodeCompleteConsumer
Abstract interface for a consumer of code-completion information.
Definition: CodeCompleteConsumer.h:1003
clang::EmitLLVMOnlyAction
Definition: CodeGenAction.h:104
clang::Sema::LocalEagerInstantiationScope
Definition: Sema.h:9455
clang::Sema::GlobalEagerInstantiationScope
Definition: Sema.h:9400
clang::IncrementalParser::getCI
const CompilerInstance * getCI() const
Definition: IncrementalParser.h:64
clang::ASTConsumer::HandleTranslationUnit
virtual void HandleTranslationUnit(ASTContext &Ctx)
HandleTranslationUnit - This method is called when the ASTs for entire translation unit have been par...
Definition: ASTConsumer.h:66
std
Definition: Format.h:4296
clang::TranslationUnitKind
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:816
clang::FrontendAction
Abstract base class for actions which can be performed by the frontend.
Definition: FrontendAction.h:36
clang
Definition: CalledOnceCheck.h:17
clang::CompilerInstance::getCodeCompletionConsumer
CodeCompleteConsumer & getCodeCompletionConsumer() const
Definition: CompilerInstance.h:560
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:38
clang::IncrementalParser::~IncrementalParser
~IncrementalParser()
Definition: IncrementalParser.cpp:137
clang::WrapperFrontendAction::WrappedAction
std::unique_ptr< FrontendAction > WrappedAction
Definition: FrontendAction.h:307
clang::PartialTranslationUnit
The class keeps track of various objects created as part of processing incremental inputs.
Definition: PartialTranslationUnit.h:29
CompilerInstance.h
clang::WrapperFrontendAction::EndSourceFile
void EndSourceFile() override
Perform any per-file post processing, deallocate per-file objects, and run statistics and output file...
Definition: FrontendAction.cpp:1173
clang::CompilerInstance::getFrontendOpts
FrontendOptions & getFrontendOpts()
Definition: CompilerInstance.h:282
clang::comments::tok::eof
@ eof
Definition: CommentLexer.h:33
clang::CompilerInstance::getDiagnostics
DiagnosticsEngine & getDiagnostics() const
Get the current diagnostics engine.
Definition: CompilerInstance.h:334
clang::Scope::DeclScope
@ DeclScope
This is a scope that can contain a declaration.
Definition: Scope.h:59
clang::CompilerInstance::createSema
void createSema(TranslationUnitKind TUKind, CodeCompleteConsumer *CompletionConsumer)
Create the Sema object to be used for parsing.
Definition: CompilerInstance.cpp:749
clang::Preprocessor
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:131
SM
#define SM(sm)
Definition: Cuda.cpp:81
clang::frontend::EmitLLVMOnly
@ EmitLLVMOnly
Generate LLVM IR, but do not emit anything.
Definition: FrontendOptions.h:70
DeclContextInternals.h
clang::PartialTranslationUnit::TUPart
TranslationUnitDecl * TUPart
Definition: PartialTranslationUnit.h:30
clang::FrontendOptions::CodeCompletionAt
ParsedSourceLocation CodeCompletionAt
If given, enable code completion at the provided location.
Definition: FrontendOptions.h:442
clang::WrapperFrontendAction
A frontend action which simply wraps some other runtime-specified frontend action.
Definition: FrontendAction.h:305
Parser.h