clang  14.0.0git
IncrementalParser.cpp
Go to the documentation of this file.
1 //===--------- IncrementalParser.cpp - Incremental Compilation -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class which performs incremental code compilation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "IncrementalParser.h"
14 
22 #include "clang/Parse/Parser.h"
23 #include "clang/Sema/Sema.h"
24 
25 #include "llvm/Option/ArgList.h"
26 #include "llvm/Support/CrashRecoveryContext.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/Support/Timer.h"
29 
30 #include <sstream>
31 
32 namespace clang {
33 
34 /// A custom action enabling the incremental processing functionality.
35 ///
36 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
37 /// sees a call to \p EndSourceFile it deletes some of the important objects
38 /// such as \p Preprocessor and \p Sema assuming no further input will come.
39 ///
40 /// \p IncrementalAction ensures it keep its underlying action's objects alive
41 /// as long as the \p IncrementalParser needs them.
42 ///
44 private:
45  bool IsTerminating = false;
46 
47 public:
48  IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
49  llvm::Error &Err)
50  : WrapperFrontendAction([&]() {
51  llvm::ErrorAsOutParameter EAO(&Err);
52  std::unique_ptr<FrontendAction> Act;
53  switch (CI.getFrontendOpts().ProgramAction) {
54  default:
55  Err = llvm::createStringError(
56  std::errc::state_not_recoverable,
57  "Driver initialization failed. "
58  "Incremental mode for action %d is not supported",
60  return Act;
61  case frontend::ASTDump:
62  LLVM_FALLTHROUGH;
63  case frontend::ASTPrint:
64  LLVM_FALLTHROUGH;
66  Act = CreateFrontendAction(CI);
67  break;
69  LLVM_FALLTHROUGH;
71  LLVM_FALLTHROUGH;
72  case frontend::EmitObj:
73  LLVM_FALLTHROUGH;
75  Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
76  break;
77  }
78  return Act;
79  }()) {}
80  FrontendAction *getWrapped() const { return WrappedAction.get(); }
82  return TU_Incremental;
83  }
84  void ExecuteAction() override {
86  assert(CI.hasPreprocessor() && "No PP!");
87 
88  // FIXME: Move the truncation aspect of this into Sema, we delayed this till
89  // here so the source manager would be initialized.
93 
94  // Use a code completion consumer?
95  CodeCompleteConsumer *CompletionConsumer = nullptr;
97  CompletionConsumer = &CI.getCodeCompletionConsumer();
98 
99  Preprocessor &PP = CI.getPreprocessor();
101  PP.EnterMainSourceFile();
102 
103  if (!CI.hasSema())
104  CI.createSema(getTranslationUnitKind(), CompletionConsumer);
105  }
106 
107  // Do not terminate after processing the input. This allows us to keep various
108  // clang objects alive and to incrementally grow the current TU.
109  void EndSourceFile() override {
110  // The WrappedAction can be nullptr if we issued an error in the ctor.
111  if (IsTerminating && getWrapped())
113  }
114 
115  void FinalizeAction() {
116  assert(!IsTerminating && "Already finalized!");
117  IsTerminating = true;
118  EndSourceFile();
119  }
120 };
121 
122 IncrementalParser::IncrementalParser(std::unique_ptr<CompilerInstance> Instance,
123  llvm::LLVMContext &LLVMCtx,
124  llvm::Error &Err)
125  : CI(std::move(Instance)) {
126  llvm::ErrorAsOutParameter EAO(&Err);
127  Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
128  if (Err)
129  return;
130  CI->ExecuteAction(*Act);
131  Consumer = &CI->getASTConsumer();
132  P.reset(
133  new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
134  P->Initialize();
135 }
136 
137 IncrementalParser::~IncrementalParser() { Act->FinalizeAction(); }
138 
140 IncrementalParser::ParseOrWrapTopLevelDecl() {
141  // Recover resources if we crash before exiting this method.
142  Sema &S = CI->getSema();
143  llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
144  Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
145  Sema::LocalEagerInstantiationScope LocalInstantiations(S);
146 
147  PTUs.emplace_back(PartialTranslationUnit());
148  PartialTranslationUnit &LastPTU = PTUs.back();
149  // Add a new PTU.
150  ASTContext &C = S.getASTContext();
151  C.addTranslationUnitDecl();
152  LastPTU.TUPart = C.getTranslationUnitDecl();
153 
154  // Skip previous eof due to last incremental input.
155  if (P->getCurToken().is(tok::eof)) {
156  P->ConsumeToken();
157  // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
158  // might want to do that around HandleEndOfTranslationUnit.
159  P->ExitScope();
160  S.CurContext = nullptr;
161  // Start a new PTU.
162  P->EnterScope(Scope::DeclScope);
163  S.ActOnTranslationUnitScope(P->getCurScope());
164  }
165 
167  for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl); !AtEOF;
168  AtEOF = P->ParseTopLevelDecl(ADecl)) {
169  // If we got a null return and something *was* parsed, ignore it. This
170  // is due to a top-level semicolon, an action override, or a parse error
171  // skipping something.
172  if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
173  return llvm::make_error<llvm::StringError>("Parsing failed. "
174  "The consumer rejected a decl",
175  std::error_code());
176  }
177 
178  DiagnosticsEngine &Diags = getCI()->getDiagnostics();
179  if (Diags.hasErrorOccurred()) {
180  TranslationUnitDecl *MostRecentTU = C.getTranslationUnitDecl();
181  TranslationUnitDecl *PreviousTU = MostRecentTU->getPreviousDecl();
182  assert(PreviousTU && "Must have a TU from the ASTContext initialization!");
183  TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
184  assert(FirstTU);
185  FirstTU->RedeclLink.setLatest(PreviousTU);
186  C.TUDecl = PreviousTU;
187  S.TUScope->setEntity(PreviousTU);
188 
189  // Clean up the lookup table
190  if (StoredDeclsMap *Map = PreviousTU->getLookupPtr()) {
191  for (auto I = Map->begin(); I != Map->end(); ++I) {
192  StoredDeclsList &List = I->second;
193  DeclContextLookupResult R = List.getLookupResult();
194  for (NamedDecl *D : R)
195  if (D->getTranslationUnitDecl() == MostRecentTU)
196  List.remove(D);
197  if (List.isNull())
198  Map->erase(I);
199  }
200  }
201 
202  // FIXME: Do not reset the pragma handlers.
203  Diags.Reset();
204  return llvm::make_error<llvm::StringError>("Parsing failed.",
205  std::error_code());
206  }
207 
208  // Process any TopLevelDecls generated by #pragma weak.
209  for (Decl *D : S.WeakTopLevelDecls()) {
210  DeclGroupRef DGR(D);
211  Consumer->HandleTopLevelDecl(DGR);
212  }
213 
214  LocalInstantiations.perform();
215  GlobalInstantiations.perform();
216 
217  Consumer->HandleTranslationUnit(C);
218 
219  return LastPTU;
220 }
221 
223  IncrementalAction *IncrAct = static_cast<IncrementalAction *>(Act);
224  FrontendAction *WrappedAct = IncrAct->getWrapped();
225  if (!WrappedAct->hasIRSupport())
226  return nullptr;
227  return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
228 }
229 
231 IncrementalParser::Parse(llvm::StringRef input) {
232  Preprocessor &PP = CI->getPreprocessor();
233  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
234 
235  std::ostringstream SourceName;
236  SourceName << "input_line_" << InputCount++;
237 
238  // Create an uninitialized memory buffer, copy code in and append "\n"
239  size_t InputSize = input.size(); // don't include trailing 0
240  // MemBuffer size should *not* include terminating zero
241  std::unique_ptr<llvm::MemoryBuffer> MB(
242  llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
243  SourceName.str()));
244  char *MBStart = const_cast<char *>(MB->getBufferStart());
245  memcpy(MBStart, input.data(), InputSize);
246  MBStart[InputSize] = '\n';
247 
248  SourceManager &SM = CI->getSourceManager();
249 
250  // FIXME: Create SourceLocation, which will allow clang to order the overload
251  // candidates for example
252  SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
253 
254  // Create FileID for the current buffer.
255  FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
256  /*LoadedOffset=*/0, NewLoc);
257 
258  // NewLoc only used for diags.
259  if (PP.EnterSourceFile(FID, /*DirLookup=*/0, NewLoc))
260  return llvm::make_error<llvm::StringError>("Parsing failed. "
261  "Cannot enter source file.",
262  std::error_code());
263 
264  auto PTU = ParseOrWrapTopLevelDecl();
265  if (!PTU)
266  return PTU.takeError();
267 
268  if (PP.getLangOpts().DelayedTemplateParsing) {
269  // Microsoft-specific:
270  // Late parsed templates can leave unswallowed "macro"-like tokens.
271  // They will seriously confuse the Parser when entering the next
272  // source file. So lex until we are EOF.
273  Token Tok;
274  do {
275  PP.Lex(Tok);
276  } while (Tok.isNot(tok::eof));
277  }
278 
279  Token AssertTok;
280  PP.Lex(AssertTok);
281  assert(AssertTok.is(tok::eof) &&
282  "Lexer must be EOF when starting incremental parse!");
283 
284  if (CodeGenerator *CG = getCodeGen(Act.get())) {
285  std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
286  CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
287  M->getContext());
288 
289  PTU->TheModule = std::move(M);
290  }
291 
292  return PTU;
293 }
294 } // end namespace clang
clang::IncrementalAction::IncrementalAction
IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx, llvm::Error &Err)
Definition: IncrementalParser.cpp:48
clang::CompilerInstance::hasSema
bool hasSema() const
Definition: CompilerInstance.h:503
clang::Preprocessor::enableIncrementalProcessing
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
Definition: Preprocessor.h:1644
clang::CodeGenerator
The primary public interface to the Clang code generator.
Definition: ModuleBuilder.h:43
clang::frontend::EmitAssembly
@ EmitAssembly
Emit a .s file.
Definition: FrontendOptions.h:58
Utils.h
clang::Preprocessor::isIncrementalProcessingEnabled
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
Definition: Preprocessor.h:1641
clang::FrontendOptions::ProgramAction
frontend::ActionKind ProgramAction
The frontend action to perform.
Definition: FrontendOptions.h:405
clang::Preprocessor::Lex
void Lex(Token &Result)
Lex the next token for this preprocessor.
Definition: Preprocessor.cpp:888
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:88
clang::WrapperFrontendAction::hasCodeCompletionSupport
bool hasCodeCompletionSupport() const override
Does this action support use with code completion?
Definition: FrontendAction.cpp:1131
clang::FrontendAction::hasIRSupport
virtual bool hasIRSupport() const
Does this action support use with IR files?
Definition: FrontendAction.h:197
clang::IncrementalAction::getTranslationUnitKind
TranslationUnitKind getTranslationUnitKind() override
For AST-based actions, the kind of translation unit we're handling.
Definition: IncrementalParser.cpp:81
AttributeLangSupport::C
@ C
Definition: SemaDeclAttr.cpp:54
clang::CreateFrontendAction
std::unique_ptr< FrontendAction > CreateFrontendAction(CompilerInstance &CI)
Construct the FrontendAction of a compiler invocation based on the options specified for the compiler...
Definition: ExecuteCompilerInvocation.cpp:137
clang::IncrementalAction::ExecuteAction
void ExecuteAction() override
Callback to run the program action, using the initialized compiler instance.
Definition: IncrementalParser.cpp:84
memcpy
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
Definition: __clang_cuda_device_functions.h:1549
clang::Parser
Parser - This implements a parser for the C family of languages.
Definition: Parser.h:60
ModuleBuilder.h
clang::frontend::ASTDump
@ ASTDump
Parse ASTs and dump them.
Definition: FrontendOptions.h:40
llvm::Expected
Definition: LLVM.h:41
FrontendAction.h
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
clang::frontend::PluginAction
@ PluginAction
Run a plugin action,.
Definition: FrontendOptions.h:109
clang::CompilerInstance::createCodeCompletionConsumer
void createCodeCompletionConsumer()
Create a code completion consumer using the invocation; note that this will cause the source manager ...
Definition: CompilerInstance.cpp:709
CodeGenAction.h
clang::IncrementalParser::IncrementalParser
IncrementalParser(std::unique_ptr< CompilerInstance > Instance, llvm::LLVMContext &LLVMCtx, llvm::Error &Err)
Definition: IncrementalParser.cpp:122
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:626
clang::ParsedSourceLocation::FileName
std::string FileName
Definition: CommandLineSourceLoc.h:25
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:960
clang::frontend::ASTPrint
@ ASTPrint
Parse ASTs and print them.
Definition: FrontendOptions.h:43
BackendUtil.h
clang::CodeGenAction
Definition: CodeGenAction.h:24
clang::frontend::EmitObj
@ EmitObj
Emit a .o file.
Definition: FrontendOptions.h:76
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:212
clang::TU_Incremental
@ TU_Incremental
The translation unit is a is a complete translation unit that we might incrementally extend later.
Definition: LangOptions.h:716
clang::frontend::ParseSyntaxOnly
@ ParseSyntaxOnly
Parse and perform semantic analysis.
Definition: FrontendOptions.h:106
clang::IncrementalParser::Parse
llvm::Expected< PartialTranslationUnit & > Parse(llvm::StringRef Input)
Parses incremental input by creating an in-memory file.
Definition: IncrementalParser.cpp:231
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:97
clang::IncrementalAction::FinalizeAction
void FinalizeAction()
Definition: IncrementalParser.cpp:115
clang::IncrementalAction::EndSourceFile
void EndSourceFile() override
Perform any per-file post processing, deallocate per-file objects, and run statistics and output file...
Definition: IncrementalParser.cpp:109
clang::FrontendAction::getCompilerInstance
CompilerInstance & getCompilerInstance() const
Definition: FrontendAction.h:119
clang::CompilerInstance::getPreprocessor
Preprocessor & getPreprocessor() const
Return the current preprocessor.
Definition: CompilerInstance.h:444
clang::SrcMgr::C_User
@ C_User
Definition: SourceManager.h:79
clang::Preprocessor::EnterMainSourceFile
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
Definition: Preprocessor.cpp:530
clang::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
clang::CompilerInstance
CompilerInstance - Helper class for managing a single instance of the Clang compiler.
Definition: CompilerInstance.h:74
Sema.h
clang::ASTConsumer::HandleTopLevelDecl
virtual bool HandleTopLevelDecl(DeclGroupRef D)
HandleTopLevelDecl - Handle the specified top-level declaration.
Definition: ASTConsumer.cpp:18
clang::CompilerInstance::hasPreprocessor
bool hasPreprocessor() const
Definition: CompilerInstance.h:441
clang::IncrementalAction
A custom action enabling the incremental processing functionality.
Definition: IncrementalParser.cpp:43
clang::getCodeGen
static CodeGenerator * getCodeGen(FrontendAction *Act)
Definition: IncrementalParser.cpp:222
clang::CompilerInstance::hasCodeCompletionConsumer
bool hasCodeCompletionConsumer() const
Definition: CompilerInstance.h:560
IncrementalParser.h
clang::IncrementalAction::getWrapped
FrontendAction * getWrapped() const
Definition: IncrementalParser.cpp:80
clang::Parser::DeclGroupPtrTy
OpaquePtr< DeclGroupRef > DeclGroupPtrTy
Definition: Parser.h:453
clang::Sema
Sema - This implements semantic analysis and AST building for C.
Definition: Sema.h:355
clang::CodeCompleteConsumer
Abstract interface for a consumer of code-completion information.
Definition: CodeCompleteConsumer.h:1000
clang::EmitLLVMOnlyAction
Definition: CodeGenAction.h:104
clang::Sema::LocalEagerInstantiationScope
Definition: Sema.h:9303
clang::Sema::GlobalEagerInstantiationScope
Definition: Sema.h:9248
clang::IncrementalParser::getCI
const CompilerInstance * getCI() const
Definition: IncrementalParser.h:65
clang::ASTConsumer::HandleTranslationUnit
virtual void HandleTranslationUnit(ASTContext &Ctx)
HandleTranslationUnit - This method is called when the ASTs for entire translation unit have been par...
Definition: ASTConsumer.h:66
std
Definition: Format.h:4034
clang::TranslationUnitKind
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:703
clang::FrontendAction
Abstract base class for actions which can be performed by the frontend.
Definition: FrontendAction.h:36
clang
Definition: CalledOnceCheck.h:17
clang::CompilerInstance::getCodeCompletionConsumer
CodeCompleteConsumer & getCodeCompletionConsumer() const
Definition: CompilerInstance.h:562
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:40
clang::IncrementalParser::~IncrementalParser
~IncrementalParser()
Definition: IncrementalParser.cpp:137
clang::WrapperFrontendAction::WrappedAction
std::unique_ptr< FrontendAction > WrappedAction
Definition: FrontendAction.h:307
clang::PartialTranslationUnit
The class keeps track of various objects created as part of processing incremental inputs.
Definition: PartialTranslationUnit.h:29
CompilerInstance.h
clang::WrapperFrontendAction::EndSourceFile
void EndSourceFile() override
Perform any per-file post processing, deallocate per-file objects, and run statistics and output file...
Definition: FrontendAction.cpp:1108
clang::CompilerInstance::getFrontendOpts
FrontendOptions & getFrontendOpts()
Definition: CompilerInstance.h:284
clang::comments::tok::eof
@ eof
Definition: CommentLexer.h:33
clang::CompilerInstance::getDiagnostics
DiagnosticsEngine & getDiagnostics() const
Get the current diagnostics engine.
Definition: CompilerInstance.h:336
clang::Scope::DeclScope
@ DeclScope
This is a scope that can contain a declaration.
Definition: Scope.h:59
clang::CompilerInstance::createSema
void createSema(TranslationUnitKind TUKind, CodeCompleteConsumer *CompletionConsumer)
Create the Sema object to be used for parsing.
Definition: CompilerInstance.cpp:748
clang::Preprocessor
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:130
SM
#define SM(sm)
Definition: Cuda.cpp:78
clang::frontend::EmitLLVMOnly
@ EmitLLVMOnly
Generate LLVM IR, but do not emit anything.
Definition: FrontendOptions.h:70
DeclContextInternals.h
clang::PartialTranslationUnit::TUPart
TranslationUnitDecl * TUPart
Definition: PartialTranslationUnit.h:30
clang::FrontendOptions::CodeCompletionAt
ParsedSourceLocation CodeCompletionAt
If given, enable code completion at the provided location.
Definition: FrontendOptions.h:402
clang::Preprocessor::EnterSourceFile
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Definition: PPLexerChange.cpp:69
clang::WrapperFrontendAction
A frontend action which simply wraps some other runtime-specified frontend action.
Definition: FrontendAction.h:305
Parser.h