IncrementalParser.cpp

Go to the documentation of this file.

//===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the class which performs incremental code compilation.
//
//===----------------------------------------------------------------------===//
 
#include "IncrementalParser.h"
#include "IncrementalAction.h"
 
#include "clang/AST/ASTContext.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/DeclContextInternals.h"
#include "clang/AST/DeclTemplate.h"
#include "clang/AST/DeclVisitor.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Interpreter/PartialTranslationUnit.h"
#include "clang/Parse/Parser.h"
#include "clang/Sema/Sema.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/Error.h"
 
#include <sstream>
 
#define DEBUG_TYPE "clang-repl"
 
namespace clang {
 
// IncrementalParser::IncrementalParser() {}
 
IncrementalParser::IncrementalParser(CompilerInstance &Instance,
                                     IncrementalAction *Act, llvm::Error &Err,
                                     std::list<PartialTranslationUnit> &PTUs)
    : S(Instance.getSema()), Act(Act), PTUs(PTUs) {
  llvm::ErrorAsOutParameter EAO(&Err);
  Consumer = &S.getASTConsumer();
  P.reset(new Parser(S.getPreprocessor(), S, /*SkipBodies=*/false));
 
  if (ExternalASTSource *External = S.getASTContext().getExternalSource())
    External->StartTranslationUnit(Consumer);
 
  P->Initialize();
}
 
IncrementalParser::~IncrementalParser() { P.reset(); }
 
llvm::Expected<TranslationUnitDecl *>
IncrementalParser::ParseOrWrapTopLevelDecl() {
  // Recover resources if we crash before exiting this method.
  llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
  Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true,
                                                           /*AtEndOfTU=*/true);
  Sema::LocalEagerInstantiationScope LocalInstantiations(S, /*AtEndOfTU=*/true);
 
  // Add a new PTU.
  ASTContext &C = S.getASTContext();
  C.addTranslationUnitDecl();
 
  // Skip previous eof due to last incremental input.
  if (P->getCurToken().is(tok::annot_repl_input_end)) {
    P->ConsumeAnyToken();
    // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
    // might want to do that around HandleEndOfTranslationUnit.
    P->ExitScope();
    S.CurContext = nullptr;
    // Start a new PTU.
    P->EnterScope(Scope::DeclScope);
    S.ActOnTranslationUnitScope(P->getCurScope());
  }
 
  Parser::DeclGroupPtrTy ADecl;
  Sema::ModuleImportState ImportState;
  for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
       AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
    if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
      return llvm::make_error<llvm::StringError>("Parsing failed. "
                                                 "The consumer rejected a decl",
                                                 std::error_code());
  }
 
  DiagnosticsEngine &Diags = S.getDiagnostics();
  if (Diags.hasErrorOccurred()) {
    CleanUpPTU(C.getTranslationUnitDecl());
 
    Diags.Reset(/*soft=*/true);
    Diags.getClient()->clear();
    return llvm::make_error<llvm::StringError>("Parsing failed.",
                                               std::error_code());
  }
 
  // Process any TopLevelDecls generated by #pragma weak.
  for (Decl *D : S.WeakTopLevelDecls()) {
    DeclGroupRef DGR(D);
    Consumer->HandleTopLevelDecl(DGR);
  }
 
  LocalInstantiations.perform();
  GlobalInstantiations.perform();
 
  Consumer->HandleTranslationUnit(C);
 
  return C.getTranslationUnitDecl();
}
 
llvm::Expected<TranslationUnitDecl *>
IncrementalParser::Parse(llvm::StringRef input) {
  Preprocessor &PP = S.getPreprocessor();
  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
 
  std::ostringstream SourceName;
  SourceName << "input_line_" << InputCount++;
 
  // Create an uninitialized memory buffer, copy code in and append "\n"
  size_t InputSize = input.size(); // don't include trailing 0
  // MemBuffer size should *not* include terminating zero
  std::unique_ptr<llvm::MemoryBuffer> MB(
      llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
                                                        SourceName.str()));
  char *MBStart = const_cast<char *>(MB->getBufferStart());
  memcpy(MBStart, input.data(), InputSize);
  MBStart[InputSize] = '\n';
 
  SourceManager &SM = S.getSourceManager();
 
  // FIXME: Create SourceLocation, which will allow clang to order the overload
  // candidates for example
  SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
 
  // Create FileID for the current buffer.
  FileID FID;
  // Create FileEntry and FileID for the current buffer.
  FileEntryRef FE = SM.getFileManager().getVirtualFileRef(
      SourceName.str(), InputSize, 0 /* mod time*/);
  SM.overrideFileContents(FE, std::move(MB));
 
  // Ensure HeaderFileInfo exists before lookup to prevent assertion
  HeaderSearch &HS = PP.getHeaderSearchInfo();
  HS.getFileInfo(FE);
 
  FID = SM.createFileID(FE, NewLoc, SrcMgr::C_User);
 
  // NewLoc only used for diags.
  if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
    return llvm::make_error<llvm::StringError>("Parsing failed. "
                                               "Cannot enter source file.",
                                               std::error_code());
 
  auto PTU = ParseOrWrapTopLevelDecl();
  if (!PTU)
    return PTU.takeError();
 
  if (PP.getLangOpts().DelayedTemplateParsing) {
    // Microsoft-specific:
    // Late parsed templates can leave unswallowed "macro"-like tokens.
    // They will seriously confuse the Parser when entering the next
    // source file. So lex until we are EOF.
    Token Tok;
    do {
      PP.Lex(Tok);
    } while (Tok.isNot(tok::annot_repl_input_end));
  } else {
    Token AssertTok;
    PP.Lex(AssertTok);
    assert(AssertTok.is(tok::annot_repl_input_end) &&
           "Lexer must be EOF when starting incremental parse!");
  }
 
  return PTU;
}
 
void IncrementalParser::withdrawMostRecentTU(
    TranslationUnitDecl *MostRecentTU) {
  TranslationUnitDecl *Prev = MostRecentTU->getPreviousDecl();
  if (!Prev)
    return;
  assert(MostRecentTU->getMostRecentDecl() == MostRecentTU &&
         "Not the most recent translation unit!");
 
  // Rebuild A -> ... -> Prev -> MostRecentTU as A -> ... -> Prev.
  MostRecentTU->getFirstDecl()->RedeclLink.setLatest(Prev);
 
  // getTranslationUnitDecl() requires the active unit to be the latest one.
  ASTContext &C = S.getASTContext();
  if (C.TraversalScope.size() == 1 && C.TraversalScope.back() == MostRecentTU)
    C.TraversalScope = {Prev};
  C.TUDecl = Prev;
}
 
/// Removes decls introduced in the discarding PTU and restores the
/// redeclaration chain to previous state.
class ASTDeclUnmerger : public DeclVisitor<ASTDeclUnmerger> {
  Sema &S;
  TranslationUnitDecl *DiscardedTU;
 
  template <typename DeclT> void withdraw(Redeclarable<DeclT> *DBase) {
    if (NamedDecl *Prev = findSurvivor(static_cast<DeclT *>(DBase)))
      unlinkRedeclChain(S.getASTContext(), DBase, Prev);
  }
 
  /// The newest declaration of whatever D redeclares that still lives outside
  /// the DiscardedTU, or null if DiscardedTU introduced the name.
  NamedDecl *findSurvivor(NamedDecl *D) const {
    for (Decl *Prev = D->getPreviousDecl(); Prev;
         Prev = Prev->getPreviousDecl())
      if (Prev->getTranslationUnitDecl() != DiscardedTU)
        return dyn_cast<NamedDecl>(Prev);
    return nullptr;
  }
 
  template <typename DeclT>
  void unlinkRedeclChain(ASTContext &C, Redeclarable<DeclT> *DBase,
                         NamedDecl *PrevND) {
    auto *Latest = static_cast<DeclT *>(DBase);
    auto *Survivor = cast<DeclT>(PrevND);
 
    // Rebuild First -> ... -> Survivor -> ... -> Latest as
    // First -> ... -> Survivor.
    Latest->getFirstDecl()->RedeclLink.setLatest(Survivor);
 
    // The chain is circular: a withdrawn declaration still linked into it can
    // never walk back around to itself, so redecls() on one would not
    // terminate. Give each withdrawn declaration a chain of its own.
    for (DeclT *Dead = Latest; Dead != Survivor;) {
      DeclT *Next = Dead->getPreviousDecl();
      Dead->First = Dead;
      Dead->RedeclLink = Redeclarable<DeclT>::LatestDeclLink(C);
      Dead = Next;
    }
  }
 
  /// Remove entry from "C"'s lookup tables
  void removeFromLookups(NamedDecl *D) {
    if (D->getDeclName().isEmpty())
      return;
 
    if (D->getDeclName().isIdentifier() && D->getDeclName().getFETokenInfo() &&
        !D->getLangOpts().ObjC && !D->getLangOpts().CPlusPlus)
      S.IdResolver.RemoveDecl(D);
 
    ExternCContextDecl *ECCD = S.getASTContext().getExternCContextDecl();
    if (StoredDeclsMap *Map = ECCD->getPrimaryContext()->getLookupPtr()) {
      auto It = Map->find(D->getDeclName());
      if (It != Map->end())
        It->second.remove(D);
    }
  }
 
  /// Remove Decls defined in this DC from the lookup table
  /// and restore the redeclaration chain to previous state
  void VisitDeclContext(DeclContext *DC) {
    llvm::SmallVector<Decl *, 8> Members(DC->decls());
    llvm::SmallVector<NamedDecl *, 8> Survivors;
    for (Decl *M : Members) {
      if (auto *ND = dyn_cast<NamedDecl>(M))
        if (NamedDecl *Prev = findSurvivor(ND))
          Survivors.push_back(Prev);
      Visit(M);          // restore redecls
      DC->removeDecl(M); // remove from lookup
      if (auto *ND = dyn_cast<NamedDecl>(M))
        removeFromLookups(ND);
    }
 
    // Restore lookup for the surviving predecessor
    // of any removed decl that had a surviving predecessor
    DeclContext *Primary = DC->getPrimaryContext();
    for (NamedDecl *Prev : Survivors)
      Primary->makeDeclVisibleInContext(Prev);
  }
 
public:
  ASTDeclUnmerger(Sema &S, TranslationUnitDecl *DiscardedTU)
      : S(S), DiscardedTU(DiscardedTU) {}
 
  void VisitDecl(Decl *D) {
    if (auto *DC = dyn_cast<DeclContext>(D))
      VisitDeclContext(DC);
  }
 
  void VisitFunctionDecl(FunctionDecl *D) { withdraw(D); }
  void VisitNamespaceAliasDecl(NamespaceAliasDecl *D) { withdraw(D); }
  void VisitTypedefNameDecl(TypedefNameDecl *D) { withdraw(D); }
  void VisitUsingShadowDecl(UsingShadowDecl *D) { withdraw(D); }
  void VisitVarDecl(VarDecl *D) { withdraw(D); }
 
  void VisitTagDecl(TagDecl *D) {
    NamedDecl *Prev = findSurvivor(D);
    if (!Prev)
      return;
    unlinkRedeclChain(S.getASTContext(), D, Prev);
 
    // A class definition is kept in DefinitionData outside the
    // redeclaration chain
    auto *RD = dyn_cast<CXXRecordDecl>(Prev);
    if (!RD)
      return;
    if (CXXRecordDecl *Def = RD->getDefinition();
        Def && Def->getTranslationUnitDecl() == DiscardedTU)
      for (auto *R : RD->redecls())
        cast<CXXRecordDecl>(R)->DefinitionData = nullptr;
  }
 
  void VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
    withdraw(D);
    Visit(D->getTemplatedDecl());
  }
 
  void VisitNamespaceDecl(NamespaceDecl *D) {
    // Handle cases of nested redeclarations like:
    // PTU1: namespace outer { namespace ns { class Foo; } }
    // PTU2: namespace outer { namespace ns { class Foo { ... }; error; } }
    // Foo's redeclaration needs to be restored
    VisitDeclContext(D);
    withdraw(D);
  }
 
  void VisitTranslationUnitDecl(TranslationUnitDecl *D) { VisitDeclContext(D); }
};
 
void IncrementalParser::CleanUpPTU(TranslationUnitDecl *MostRecentTU) {
  ASTDeclUnmerger(S, MostRecentTU).Visit(MostRecentTU);
 
  // Lookup alone is not enough: the redeclaration chain still reaches these.
  withdrawMostRecentTU(MostRecentTU);
}
 
PartialTranslationUnit &
IncrementalParser::RegisterPTU(TranslationUnitDecl *TU,
                               std::unique_ptr<llvm::Module> M /*={}*/) {
  PTUs.emplace_back(PartialTranslationUnit());
  PartialTranslationUnit &LastPTU = PTUs.back();
  LastPTU.TUPart = TU;
 
  if (!M)
    M = Act->GenModule();
 
  assert((!Act->getCodeGen() || M) && "Must have a llvm::Module at this point");
 
  LastPTU.TheModule = std::move(M);
  LLVM_DEBUG(llvm::dbgs() << "compile-ptu " << PTUs.size() - 1
                          << ": [TU=" << LastPTU.TUPart);
  if (LastPTU.TheModule)
    LLVM_DEBUG(llvm::dbgs() << ", M=" << LastPTU.TheModule.get() << " ("
                            << LastPTU.TheModule->getName() << ")");
  LLVM_DEBUG(llvm::dbgs() << "]\n");
  return LastPTU;
}
} // end namespace clang