clang  15.0.0git
Preprocessor.cpp
Go to the documentation of this file.
1 //===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Preprocessor interface.
10 //
11 //===----------------------------------------------------------------------===//
12 //
13 // Options to support:
14 // -H - Print the name of each header file used.
15 // -d[DNI] - Dump various things.
16 // -fworking-directory - #line's with preprocessor's working dir.
17 // -fpreprocessed
18 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19 // -W*
20 // -w
21 //
22 // Messages to emit:
23 // "Multiple include guards may be useful for:\n"
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "clang/Lex/Preprocessor.h"
28 #include "clang/Basic/Builtins.h"
32 #include "clang/Basic/LLVM.h"
34 #include "clang/Basic/Module.h"
37 #include "clang/Basic/TargetInfo.h"
40 #include "clang/Lex/HeaderSearch.h"
42 #include "clang/Lex/Lexer.h"
44 #include "clang/Lex/MacroArgs.h"
45 #include "clang/Lex/MacroInfo.h"
46 #include "clang/Lex/ModuleLoader.h"
47 #include "clang/Lex/Pragma.h"
52 #include "clang/Lex/Token.h"
53 #include "clang/Lex/TokenLexer.h"
54 #include "llvm/ADT/APInt.h"
55 #include "llvm/ADT/ArrayRef.h"
56 #include "llvm/ADT/DenseMap.h"
57 #include "llvm/ADT/STLExtras.h"
58 #include "llvm/ADT/SmallString.h"
59 #include "llvm/ADT/SmallVector.h"
60 #include "llvm/ADT/StringRef.h"
61 #include "llvm/ADT/StringSwitch.h"
62 #include "llvm/Support/Capacity.h"
63 #include "llvm/Support/ErrorHandling.h"
64 #include "llvm/Support/MemoryBuffer.h"
65 #include "llvm/Support/raw_ostream.h"
66 #include <algorithm>
67 #include <cassert>
68 #include <memory>
69 #include <string>
70 #include <utility>
71 #include <vector>
72 
73 using namespace clang;
74 
75 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
76 
78 
79 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
80  DiagnosticsEngine &diags, LangOptions &opts,
81  SourceManager &SM, HeaderSearch &Headers,
82  ModuleLoader &TheModuleLoader,
83  IdentifierInfoLookup *IILookup, bool OwnsHeaders,
84  TranslationUnitKind TUKind)
85  : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86  FileMgr(Headers.getFileMgr()), SourceMgr(SM),
87  ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88  TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
89  // As the language options may have not been loaded yet (when
90  // deserializing an ASTUnit), adding keywords to the identifier table is
91  // deferred to Preprocessor::Initialize().
92  Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
93  TUKind(TUKind), SkipMainFilePreamble(0, true),
94  CurSubmoduleState(&NullSubmoduleState) {
95  OwnsHeaderSearch = OwnsHeaders;
96 
97  // Default to discarding comments.
98  KeepComments = false;
99  KeepMacroComments = false;
100  SuppressIncludeNotFoundError = false;
101 
102  // Macro expansion is enabled.
103  DisableMacroExpansion = false;
104  MacroExpansionInDirectivesOverride = false;
105  InMacroArgs = false;
106  ArgMacro = nullptr;
107  InMacroArgPreExpansion = false;
108  NumCachedTokenLexers = 0;
109  PragmasEnabled = true;
110  ParsingIfOrElifDirective = false;
111  PreprocessedOutput = false;
112 
113  // We haven't read anything from the external source.
114  ReadMacrosFromExternalSource = false;
115 
116  BuiltinInfo = std::make_unique<Builtin::Context>();
117 
118  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
119  // a macro. They get unpoisoned where it is allowed.
120  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
121  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
122  (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
123  SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
124 
125  // Initialize the pragma handlers.
126  RegisterBuiltinPragmas();
127 
128  // Initialize builtin macros like __LINE__ and friends.
129  RegisterBuiltinMacros();
130 
131  if(LangOpts.Borland) {
132  Ident__exception_info = getIdentifierInfo("_exception_info");
133  Ident___exception_info = getIdentifierInfo("__exception_info");
134  Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
135  Ident__exception_code = getIdentifierInfo("_exception_code");
136  Ident___exception_code = getIdentifierInfo("__exception_code");
137  Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
138  Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
139  Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
140  Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
141  } else {
142  Ident__exception_info = Ident__exception_code = nullptr;
143  Ident__abnormal_termination = Ident___exception_info = nullptr;
144  Ident___exception_code = Ident___abnormal_termination = nullptr;
145  Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
146  Ident_AbnormalTermination = nullptr;
147  }
148 
149  // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
151  SkippingUntilPragmaHdrStop = true;
152 
153  // If using a PCH with a through header, start skipping tokens.
154  if (!this->PPOpts->PCHThroughHeader.empty() &&
155  !this->PPOpts->ImplicitPCHInclude.empty())
156  SkippingUntilPCHThroughHeader = true;
157 
158  if (this->PPOpts->GeneratePreamble)
159  PreambleConditionalStack.startRecording();
160 
161  ExcludedConditionalDirectiveSkipMappings =
162  this->PPOpts->ExcludedConditionalDirectiveSkipMappings;
163  if (ExcludedConditionalDirectiveSkipMappings)
164  ExcludedConditionalDirectiveSkipMappings->clear();
165 
166  MaxTokens = LangOpts.MaxTokens;
167 }
168 
170  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
171 
172  IncludeMacroStack.clear();
173 
174  // Destroy any macro definitions.
175  while (MacroInfoChain *I = MIChainHead) {
176  MIChainHead = I->Next;
177  I->~MacroInfoChain();
178  }
179 
180  // Free any cached macro expanders.
181  // This populates MacroArgCache, so all TokenLexers need to be destroyed
182  // before the code below that frees up the MacroArgCache list.
183  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
184  CurTokenLexer.reset();
185 
186  // Free any cached MacroArgs.
187  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
188  ArgList = ArgList->deallocate();
189 
190  // Delete the header search info, if we own it.
191  if (OwnsHeaderSearch)
192  delete &HeaderInfo;
193 }
194 
196  const TargetInfo *AuxTarget) {
197  assert((!this->Target || this->Target == &Target) &&
198  "Invalid override of target information");
199  this->Target = &Target;
200 
201  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
202  "Invalid override of aux target information.");
203  this->AuxTarget = AuxTarget;
204 
205  // Initialize information about built-ins.
206  BuiltinInfo->InitializeTarget(Target, AuxTarget);
207  HeaderInfo.setTarget(Target);
208 
209  // Populate the identifier table with info about keywords for the current language.
210  Identifiers.AddKeywords(LangOpts);
211 
212  // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
213  setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
214 }
215 
217  NumEnteredSourceFiles = 0;
218 
219  // Reset pragmas
220  PragmaHandlersBackup = std::move(PragmaHandlers);
221  PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
222  RegisterBuiltinPragmas();
223 
224  // Reset PredefinesFileID
225  PredefinesFileID = FileID();
226 }
227 
229  NumEnteredSourceFiles = 1;
230 
231  PragmaHandlers = std::move(PragmaHandlersBackup);
232 }
233 
234 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
235  llvm::errs() << tok::getTokenName(Tok.getKind());
236 
237  if (!Tok.isAnnotation())
238  llvm::errs() << " '" << getSpelling(Tok) << "'";
239 
240  if (!DumpFlags) return;
241 
242  llvm::errs() << "\t";
243  if (Tok.isAtStartOfLine())
244  llvm::errs() << " [StartOfLine]";
245  if (Tok.hasLeadingSpace())
246  llvm::errs() << " [LeadingSpace]";
247  if (Tok.isExpandDisabled())
248  llvm::errs() << " [ExpandDisabled]";
249  if (Tok.needsCleaning()) {
250  const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
251  llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
252  << "']";
253  }
254 
255  llvm::errs() << "\tLoc=<";
256  DumpLocation(Tok.getLocation());
257  llvm::errs() << ">";
258 }
259 
261  Loc.print(llvm::errs(), SourceMgr);
262 }
263 
264 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
265  llvm::errs() << "MACRO: ";
266  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
268  llvm::errs() << " ";
269  }
270  llvm::errs() << "\n";
271 }
272 
274  llvm::errs() << "\n*** Preprocessor Stats:\n";
275  llvm::errs() << NumDirectives << " directives found:\n";
276  llvm::errs() << " " << NumDefined << " #define.\n";
277  llvm::errs() << " " << NumUndefined << " #undef.\n";
278  llvm::errs() << " #include/#include_next/#import:\n";
279  llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
280  llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
281  llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
282  llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
283  llvm::errs() << " " << NumEndif << " #endif.\n";
284  llvm::errs() << " " << NumPragma << " #pragma.\n";
285  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
286 
287  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
288  << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
289  << NumFastMacroExpanded << " on the fast path.\n";
290  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
291  << " token paste (##) operations performed, "
292  << NumFastTokenPaste << " on the fast path.\n";
293 
294  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
295 
296  llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
297  llvm::errs() << "\n Macro Expanded Tokens: "
298  << llvm::capacity_in_bytes(MacroExpandedTokens);
299  llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
300  // FIXME: List information for all submodules.
301  llvm::errs() << "\n Macros: "
302  << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
303  llvm::errs() << "\n #pragma push_macro Info: "
304  << llvm::capacity_in_bytes(PragmaPushMacroInfo);
305  llvm::errs() << "\n Poison Reasons: "
306  << llvm::capacity_in_bytes(PoisonReasons);
307  llvm::errs() << "\n Comment Handlers: "
308  << llvm::capacity_in_bytes(CommentHandlers) << "\n";
309 }
310 
312 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
313  if (IncludeExternalMacros && ExternalSource &&
314  !ReadMacrosFromExternalSource) {
315  ReadMacrosFromExternalSource = true;
316  ExternalSource->ReadDefinedMacros();
317  }
318 
319  // Make sure we cover all macros in visible modules.
320  for (const ModuleMacro &Macro : ModuleMacros)
321  CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
322 
323  return CurSubmoduleState->Macros.begin();
324 }
325 
327  return BP.getTotalMemory()
328  + llvm::capacity_in_bytes(MacroExpandedTokens)
329  + Predefines.capacity() /* Predefines buffer. */
330  // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
331  // and ModuleMacros.
332  + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
333  + llvm::capacity_in_bytes(PragmaPushMacroInfo)
334  + llvm::capacity_in_bytes(PoisonReasons)
335  + llvm::capacity_in_bytes(CommentHandlers);
336 }
337 
339 Preprocessor::macro_end(bool IncludeExternalMacros) const {
340  if (IncludeExternalMacros && ExternalSource &&
341  !ReadMacrosFromExternalSource) {
342  ReadMacrosFromExternalSource = true;
343  ExternalSource->ReadDefinedMacros();
344  }
345 
346  return CurSubmoduleState->Macros.end();
347 }
348 
349 /// Compares macro tokens with a specified token value sequence.
350 static bool MacroDefinitionEquals(const MacroInfo *MI,
351  ArrayRef<TokenValue> Tokens) {
352  return Tokens.size() == MI->getNumTokens() &&
353  std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
354 }
355 
357  SourceLocation Loc,
358  ArrayRef<TokenValue> Tokens) const {
359  SourceLocation BestLocation;
360  StringRef BestSpelling;
362  I != E; ++I) {
364  Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
365  if (!Def || !Def.getMacroInfo())
366  continue;
367  if (!Def.getMacroInfo()->isObjectLike())
368  continue;
369  if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
370  continue;
371  SourceLocation Location = Def.getLocation();
372  // Choose the macro defined latest.
373  if (BestLocation.isInvalid() ||
374  (Location.isValid() &&
375  SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
376  BestLocation = Location;
377  BestSpelling = I->first->getName();
378  }
379  }
380  return BestSpelling;
381 }
382 
384  if (CurLexer)
385  CurLexerKind = CLK_Lexer;
386  else if (CurTokenLexer)
387  CurLexerKind = CLK_TokenLexer;
388  else
389  CurLexerKind = CLK_CachingLexer;
390 }
391 
393  unsigned CompleteLine,
394  unsigned CompleteColumn) {
395  assert(File);
396  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
397  assert(!CodeCompletionFile && "Already set");
398 
399  // Load the actual file's contents.
402  if (!Buffer)
403  return true;
404 
405  // Find the byte position of the truncation point.
406  const char *Position = Buffer->getBufferStart();
407  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
408  for (; *Position; ++Position) {
409  if (*Position != '\r' && *Position != '\n')
410  continue;
411 
412  // Eat \r\n or \n\r as a single line.
413  if ((Position[1] == '\r' || Position[1] == '\n') &&
414  Position[0] != Position[1])
415  ++Position;
416  ++Position;
417  break;
418  }
419  }
420 
421  Position += CompleteColumn - 1;
422 
423  // If pointing inside the preamble, adjust the position at the beginning of
424  // the file after the preamble.
425  if (SkipMainFilePreamble.first &&
426  SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
427  if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
428  Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
429  }
430 
431  if (Position > Buffer->getBufferEnd())
432  Position = Buffer->getBufferEnd();
433 
434  CodeCompletionFile = File;
435  CodeCompletionOffset = Position - Buffer->getBufferStart();
436 
437  auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
438  Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
439  char *NewBuf = NewBuffer->getBufferStart();
440  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
441  *NewPos = '\0';
442  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
443  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
444 
445  return false;
446 }
447 
448 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
449  bool IsAngled) {
451  if (CodeComplete)
452  CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
453 }
454 
457  if (CodeComplete)
458  CodeComplete->CodeCompleteNaturalLanguage();
459 }
460 
461 /// getSpelling - This method is used to get the spelling of a token into a
462 /// SmallVector. Note that the returned StringRef may not point to the
463 /// supplied buffer if a copy can be avoided.
464 StringRef Preprocessor::getSpelling(const Token &Tok,
465  SmallVectorImpl<char> &Buffer,
466  bool *Invalid) const {
467  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
468  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
469  // Try the fast path.
470  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
471  return II->getName();
472  }
473 
474  // Resize the buffer if we need to copy into it.
475  if (Tok.needsCleaning())
476  Buffer.resize(Tok.getLength());
477 
478  const char *Ptr = Buffer.data();
479  unsigned Len = getSpelling(Tok, Ptr, Invalid);
480  return StringRef(Ptr, Len);
481 }
482 
483 /// CreateString - Plop the specified string into a scratch buffer and return a
484 /// location for it. If specified, the source location provides a source
485 /// location for the token.
486 void Preprocessor::CreateString(StringRef Str, Token &Tok,
487  SourceLocation ExpansionLocStart,
488  SourceLocation ExpansionLocEnd) {
489  Tok.setLength(Str.size());
490 
491  const char *DestPtr;
492  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
493 
494  if (ExpansionLocStart.isValid())
495  Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
496  ExpansionLocEnd, Str.size());
497  Tok.setLocation(Loc);
498 
499  // If this is a raw identifier or a literal token, set the pointer data.
500  if (Tok.is(tok::raw_identifier))
501  Tok.setRawIdentifierData(DestPtr);
502  else if (Tok.isLiteral())
503  Tok.setLiteralData(DestPtr);
504 }
505 
507  auto &SM = getSourceManager();
508  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
509  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
510  bool Invalid = false;
511  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
512  if (Invalid)
513  return SourceLocation();
514 
515  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
516  const char *DestPtr;
517  SourceLocation Spelling =
518  ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
519  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
520 }
521 
523  if (!getLangOpts().isCompilingModule())
524  return nullptr;
525 
526  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
527 }
528 
529 //===----------------------------------------------------------------------===//
530 // Preprocessor Initialization Methods
531 //===----------------------------------------------------------------------===//
532 
533 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
534 /// which implicitly adds the builtin defines etc.
536  // We do not allow the preprocessor to reenter the main file. Doing so will
537  // cause FileID's to accumulate information from both runs (e.g. #line
538  // information) and predefined macros aren't guaranteed to be set properly.
539  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
540  FileID MainFileID = SourceMgr.getMainFileID();
541 
542  // If MainFileID is loaded it means we loaded an AST file, no need to enter
543  // a main file.
544  if (!SourceMgr.isLoadedFileID(MainFileID)) {
545  // Enter the main file source buffer.
546  EnterSourceFile(MainFileID, nullptr, SourceLocation());
547 
548  // If we've been asked to skip bytes in the main file (e.g., as part of a
549  // precompiled preamble), do so now.
550  if (SkipMainFilePreamble.first > 0)
551  CurLexer->SetByteOffset(SkipMainFilePreamble.first,
552  SkipMainFilePreamble.second);
553 
554  // Tell the header info that the main file was entered. If the file is later
555  // #imported, it won't be re-entered.
556  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
557  markIncluded(FE);
558  }
559 
560  // Preprocess Predefines to populate the initial preprocessor state.
561  std::unique_ptr<llvm::MemoryBuffer> SB =
562  llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
563  assert(SB && "Cannot create predefined source buffer");
564  FileID FID = SourceMgr.createFileID(std::move(SB));
565  assert(FID.isValid() && "Could not create FileID for predefines?");
566  setPredefinesFileID(FID);
567 
568  // Start parsing the predefines.
569  EnterSourceFile(FID, nullptr, SourceLocation());
570 
571  if (!PPOpts->PCHThroughHeader.empty()) {
572  // Lookup and save the FileID for the through header. If it isn't found
573  // in the search path, it's a fatal error.
575  SourceLocation(), PPOpts->PCHThroughHeader,
576  /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
577  /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
578  /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
579  /*IsFrameworkFound=*/nullptr);
580  if (!File) {
581  Diag(SourceLocation(), diag::err_pp_through_header_not_found)
582  << PPOpts->PCHThroughHeader;
583  return;
584  }
585  setPCHThroughHeaderFileID(
587  }
588 
589  // Skip tokens from the Predefines and if needed the main file.
590  if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
591  (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
593 }
594 
595 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
596  assert(PCHThroughHeaderFileID.isInvalid() &&
597  "PCHThroughHeaderFileID already set!");
598  PCHThroughHeaderFileID = FID;
599 }
600 
602  assert(PCHThroughHeaderFileID.isValid() &&
603  "Invalid PCH through header FileID");
604  return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
605 }
606 
608  return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
609  PCHThroughHeaderFileID.isValid();
610 }
611 
613  return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
614  PCHThroughHeaderFileID.isValid();
615 }
616 
618  return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
619 }
620 
622  return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
623 }
624 
625 /// Skip tokens until after the #include of the through header or
626 /// until after a #pragma hdrstop is seen. Tokens in the predefines file
627 /// and the main file may be skipped. If the end of the predefines file
628 /// is reached, skipping continues into the main file. If the end of the
629 /// main file is reached, it's a fatal error.
631  bool ReachedMainFileEOF = false;
632  bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
633  bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
634  Token Tok;
635  while (true) {
636  bool InPredefines =
637  (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
638  switch (CurLexerKind) {
639  case CLK_Lexer:
640  CurLexer->Lex(Tok);
641  break;
642  case CLK_TokenLexer:
643  CurTokenLexer->Lex(Tok);
644  break;
645  case CLK_CachingLexer:
646  CachingLex(Tok);
647  break;
648  case CLK_LexAfterModuleImport:
650  break;
651  }
652  if (Tok.is(tok::eof) && !InPredefines) {
653  ReachedMainFileEOF = true;
654  break;
655  }
656  if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
657  break;
658  if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
659  break;
660  }
661  if (ReachedMainFileEOF) {
662  if (UsingPCHThroughHeader)
663  Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
664  << PPOpts->PCHThroughHeader << 1;
665  else if (!PPOpts->PCHWithHdrStopCreate)
666  Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
667  }
668 }
669 
670 void Preprocessor::replayPreambleConditionalStack() {
671  // Restore the conditional stack from the preamble, if there is one.
672  if (PreambleConditionalStack.isReplaying()) {
673  assert(CurPPLexer &&
674  "CurPPLexer is null when calling replayPreambleConditionalStack.");
675  CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
676  PreambleConditionalStack.doneReplaying();
677  if (PreambleConditionalStack.reachedEOFWhileSkipping())
678  SkipExcludedConditionalBlock(
679  PreambleConditionalStack.SkipInfo->HashTokenLoc,
680  PreambleConditionalStack.SkipInfo->IfTokenLoc,
681  PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
682  PreambleConditionalStack.SkipInfo->FoundElse,
683  PreambleConditionalStack.SkipInfo->ElseLoc);
684  }
685 }
686 
688  // Notify the client that we reached the end of the source file.
689  if (Callbacks)
690  Callbacks->EndOfMainFile();
691 }
692 
693 //===----------------------------------------------------------------------===//
694 // Lexer Event Handling.
695 //===----------------------------------------------------------------------===//
696 
697 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
698 /// identifier information for the token and install it into the token,
699 /// updating the token kind accordingly.
701  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
702 
703  // Look up this token, see if it is a macro, or if it is a language keyword.
704  IdentifierInfo *II;
705  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
706  // No cleaning needed, just use the characters from the lexed buffer.
707  II = getIdentifierInfo(Identifier.getRawIdentifier());
708  } else {
709  // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
710  SmallString<64> IdentifierBuffer;
711  StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
712 
713  if (Identifier.hasUCN()) {
714  SmallString<64> UCNIdentifierBuffer;
715  expandUCNs(UCNIdentifierBuffer, CleanedStr);
716  II = getIdentifierInfo(UCNIdentifierBuffer);
717  } else {
718  II = getIdentifierInfo(CleanedStr);
719  }
720  }
721 
722  // Update the token info (identifier info and appropriate token kind).
723  // FIXME: the raw_identifier may contain leading whitespace which is removed
724  // from the cleaned identifier token. The SourceLocation should be updated to
725  // refer to the non-whitespace character. For instance, the text "\\\nB" (a
726  // line continuation before 'B') is parsed as a single tok::raw_identifier and
727  // is cleaned to tok::identifier "B". After cleaning the token's length is
728  // still 3 and the SourceLocation refers to the location of the backslash.
729  Identifier.setIdentifierInfo(II);
730  Identifier.setKind(II->getTokenID());
731 
732  return II;
733 }
734 
735 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
736  PoisonReasons[II] = DiagID;
737 }
738 
740  assert(Ident__exception_code && Ident__exception_info);
741  assert(Ident___exception_code && Ident___exception_info);
742  Ident__exception_code->setIsPoisoned(Poison);
743  Ident___exception_code->setIsPoisoned(Poison);
744  Ident_GetExceptionCode->setIsPoisoned(Poison);
745  Ident__exception_info->setIsPoisoned(Poison);
746  Ident___exception_info->setIsPoisoned(Poison);
747  Ident_GetExceptionInfo->setIsPoisoned(Poison);
748  Ident__abnormal_termination->setIsPoisoned(Poison);
749  Ident___abnormal_termination->setIsPoisoned(Poison);
750  Ident_AbnormalTermination->setIsPoisoned(Poison);
751 }
752 
754  assert(Identifier.getIdentifierInfo() &&
755  "Can't handle identifiers without identifier info!");
756  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
757  PoisonReasons.find(Identifier.getIdentifierInfo());
758  if(it == PoisonReasons.end())
759  Diag(Identifier, diag::err_pp_used_poisoned_id);
760  else
761  Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
762 }
763 
764 /// Returns a diagnostic message kind for reporting a future keyword as
765 /// appropriate for the identifier and specified language.
767  const LangOptions &LangOpts) {
768  assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
769 
770  if (LangOpts.CPlusPlus)
771  return llvm::StringSwitch<diag::kind>(II.getName())
772 #define CXX11_KEYWORD(NAME, FLAGS) \
773  .Case(#NAME, diag::warn_cxx11_keyword)
774 #define CXX20_KEYWORD(NAME, FLAGS) \
775  .Case(#NAME, diag::warn_cxx20_keyword)
776 #include "clang/Basic/TokenKinds.def"
777  // char8_t is not modeled as a CXX20_KEYWORD because it's not
778  // unconditionally enabled in C++20 mode. (It can be disabled
779  // by -fno-char8_t.)
780  .Case("char8_t", diag::warn_cxx20_keyword)
781  ;
782 
783  llvm_unreachable(
784  "Keyword not known to come from a newer Standard or proposed Standard");
785 }
786 
787 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
788  assert(II.isOutOfDate() && "not out of date");
790 }
791 
792 /// HandleIdentifier - This callback is invoked when the lexer reads an
793 /// identifier. This callback looks up the identifier in the map and/or
794 /// potentially macro expands it or turns it into a named token (like 'for').
795 ///
796 /// Note that callers of this method are guarded by checking the
797 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
798 /// IdentifierInfo methods that compute these properties will need to change to
799 /// match.
801  assert(Identifier.getIdentifierInfo() &&
802  "Can't handle identifiers without identifier info!");
803 
804  IdentifierInfo &II = *Identifier.getIdentifierInfo();
805 
806  // If the information about this identifier is out of date, update it from
807  // the external source.
808  // We have to treat __VA_ARGS__ in a special way, since it gets
809  // serialized with isPoisoned = true, but our preprocessor may have
810  // unpoisoned it if we're defining a C99 macro.
811  if (II.isOutOfDate()) {
812  bool CurrentIsPoisoned = false;
813  const bool IsSpecialVariadicMacro =
814  &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
815  if (IsSpecialVariadicMacro)
816  CurrentIsPoisoned = II.isPoisoned();
817 
818  updateOutOfDateIdentifier(II);
819  Identifier.setKind(II.getTokenID());
820 
821  if (IsSpecialVariadicMacro)
822  II.setIsPoisoned(CurrentIsPoisoned);
823  }
824 
825  // If this identifier was poisoned, and if it was not produced from a macro
826  // expansion, emit an error.
827  if (II.isPoisoned() && CurPPLexer) {
829  }
830 
831  // If this is a macro to be expanded, do it.
832  if (MacroDefinition MD = getMacroDefinition(&II)) {
833  auto *MI = MD.getMacroInfo();
834  assert(MI && "macro definition with no macro info?");
835  if (!DisableMacroExpansion) {
836  if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
837  // C99 6.10.3p10: If the preprocessing token immediately after the
838  // macro name isn't a '(', this macro should not be expanded.
839  if (!MI->isFunctionLike() || isNextPPTokenLParen())
840  return HandleMacroExpandedIdentifier(Identifier, MD);
841  } else {
842  // C99 6.10.3.4p2 says that a disabled macro may never again be
843  // expanded, even if it's in a context where it could be expanded in the
844  // future.
846  if (MI->isObjectLike() || isNextPPTokenLParen())
847  Diag(Identifier, diag::pp_disabled_macro_expansion);
848  }
849  }
850  }
851 
852  // If this identifier is a keyword in a newer Standard or proposed Standard,
853  // produce a warning. Don't warn if we're not considering macro expansion,
854  // since this identifier might be the name of a macro.
855  // FIXME: This warning is disabled in cases where it shouldn't be, like
856  // "#define constexpr constexpr", "int constexpr;"
857  if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
859  << II.getName();
860  // Don't diagnose this keyword again in this translation unit.
861  II.setIsFutureCompatKeyword(false);
862  }
863 
864  // If this is an extension token, diagnose its use.
865  // We avoid diagnosing tokens that originate from macro definitions.
866  // FIXME: This warning is disabled in cases where it shouldn't be,
867  // like "#define TY typeof", "TY(1) x".
868  if (II.isExtensionToken() && !DisableMacroExpansion)
869  Diag(Identifier, diag::ext_token_used);
870 
871  // If this is the 'import' contextual keyword following an '@', note
872  // that the next token indicates a module name.
873  //
874  // Note that we do not treat 'import' as a contextual
875  // keyword when we're in a caching lexer, because caching lexers only get
876  // used in contexts where import declarations are disallowed.
877  //
878  // Likewise if this is the C++ Modules TS import keyword.
879  if (((LastTokenWasAt && II.isModulesImport()) ||
880  Identifier.is(tok::kw_import)) &&
881  !InMacroArgs && !DisableMacroExpansion &&
882  (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
883  CurLexerKind != CLK_CachingLexer) {
884  ModuleImportLoc = Identifier.getLocation();
885  ModuleImportPath.clear();
886  ModuleImportExpectsIdentifier = true;
887  CurLexerKind = CLK_LexAfterModuleImport;
888  }
889  return true;
890 }
891 
892 void Preprocessor::Lex(Token &Result) {
893  ++LexLevel;
894 
895  // We loop here until a lex function returns a token; this avoids recursion.
896  bool ReturnedToken;
897  do {
898  switch (CurLexerKind) {
899  case CLK_Lexer:
900  ReturnedToken = CurLexer->Lex(Result);
901  break;
902  case CLK_TokenLexer:
903  ReturnedToken = CurTokenLexer->Lex(Result);
904  break;
905  case CLK_CachingLexer:
906  CachingLex(Result);
907  ReturnedToken = true;
908  break;
909  case CLK_LexAfterModuleImport:
910  ReturnedToken = LexAfterModuleImport(Result);
911  break;
912  }
913  } while (!ReturnedToken);
914 
915  if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure)
916  return;
917 
918  if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
919  // Remember the identifier before code completion token.
920  setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
922  // Set IdenfitierInfo to null to avoid confusing code that handles both
923  // identifiers and completion tokens.
924  Result.setIdentifierInfo(nullptr);
925  }
926 
927  // Update ImportSeqState to track our position within a C++20 import-seq
928  // if this token is being produced as a result of phase 4 of translation.
929  if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
930  !Result.getFlag(Token::IsReinjected)) {
931  switch (Result.getKind()) {
932  case tok::l_paren: case tok::l_square: case tok::l_brace:
933  ImportSeqState.handleOpenBracket();
934  break;
935  case tok::r_paren: case tok::r_square:
936  ImportSeqState.handleCloseBracket();
937  break;
938  case tok::r_brace:
939  ImportSeqState.handleCloseBrace();
940  break;
941  case tok::semi:
942  ImportSeqState.handleSemi();
943  break;
944  case tok::header_name:
945  case tok::annot_header_unit:
946  ImportSeqState.handleHeaderName();
947  break;
948  case tok::kw_export:
949  ImportSeqState.handleExport();
950  break;
951  case tok::identifier:
952  if (Result.getIdentifierInfo()->isModulesImport()) {
953  ImportSeqState.handleImport();
954  if (ImportSeqState.afterImportSeq()) {
955  ModuleImportLoc = Result.getLocation();
956  ModuleImportPath.clear();
957  ModuleImportExpectsIdentifier = true;
958  CurLexerKind = CLK_LexAfterModuleImport;
959  }
960  break;
961  }
962  LLVM_FALLTHROUGH;
963  default:
964  ImportSeqState.handleMisc();
965  break;
966  }
967  }
968 
969  LastTokenWasAt = Result.is(tok::at);
970  --LexLevel;
971 
972  if ((LexLevel == 0 || PreprocessToken) &&
973  !Result.getFlag(Token::IsReinjected)) {
974  if (LexLevel == 0)
975  ++TokenCount;
976  if (OnToken)
977  OnToken(Result);
978  }
979 }
980 
981 /// Lex a header-name token (including one formed from header-name-tokens if
982 /// \p AllowConcatenation is \c true).
983 ///
984 /// \param FilenameTok Filled in with the next token. On success, this will
985 /// be either a header_name token. On failure, it will be whatever other
986 /// token was found instead.
987 /// \param AllowMacroExpansion If \c true, allow the header name to be formed
988 /// by macro expansion (concatenating tokens as necessary if the first
989 /// token is a '<').
990 /// \return \c true if we reached EOD or EOF while looking for a > token in
991 /// a concatenated header name and diagnosed it. \c false otherwise.
992 bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
993  // Lex using header-name tokenization rules if tokens are being lexed from
994  // a file. Just grab a token normally if we're in a macro expansion.
995  if (CurPPLexer)
996  CurPPLexer->LexIncludeFilename(FilenameTok);
997  else
998  Lex(FilenameTok);
999 
1000  // This could be a <foo/bar.h> file coming from a macro expansion. In this
1001  // case, glue the tokens together into an angle_string_literal token.
1002  SmallString<128> FilenameBuffer;
1003  if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
1004  bool StartOfLine = FilenameTok.isAtStartOfLine();
1005  bool LeadingSpace = FilenameTok.hasLeadingSpace();
1006  bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1007 
1008  SourceLocation Start = FilenameTok.getLocation();
1010  FilenameBuffer.push_back('<');
1011 
1012  // Consume tokens until we find a '>'.
1013  // FIXME: A header-name could be formed starting or ending with an
1014  // alternative token. It's not clear whether that's ill-formed in all
1015  // cases.
1016  while (FilenameTok.isNot(tok::greater)) {
1017  Lex(FilenameTok);
1018  if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
1019  Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
1020  Diag(Start, diag::note_matching) << tok::less;
1021  return true;
1022  }
1023 
1024  End = FilenameTok.getLocation();
1025 
1026  // FIXME: Provide code completion for #includes.
1027  if (FilenameTok.is(tok::code_completion)) {
1029  Lex(FilenameTok);
1030  continue;
1031  }
1032 
1033  // Append the spelling of this token to the buffer. If there was a space
1034  // before it, add it now.
1035  if (FilenameTok.hasLeadingSpace())
1036  FilenameBuffer.push_back(' ');
1037 
1038  // Get the spelling of the token, directly into FilenameBuffer if
1039  // possible.
1040  size_t PreAppendSize = FilenameBuffer.size();
1041  FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
1042 
1043  const char *BufPtr = &FilenameBuffer[PreAppendSize];
1044  unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
1045 
1046  // If the token was spelled somewhere else, copy it into FilenameBuffer.
1047  if (BufPtr != &FilenameBuffer[PreAppendSize])
1048  memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1049 
1050  // Resize FilenameBuffer to the correct size.
1051  if (FilenameTok.getLength() != ActualLen)
1052  FilenameBuffer.resize(PreAppendSize + ActualLen);
1053  }
1054 
1055  FilenameTok.startToken();
1056  FilenameTok.setKind(tok::header_name);
1057  FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine);
1058  FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace);
1059  FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro);
1060  CreateString(FilenameBuffer, FilenameTok, Start, End);
1061  } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
1062  // Convert a string-literal token of the form " h-char-sequence "
1063  // (produced by macro expansion) into a header-name token.
1064  //
1065  // The rules for header-names don't quite match the rules for
1066  // string-literals, but all the places where they differ result in
1067  // undefined behavior, so we can and do treat them the same.
1068  //
1069  // A string-literal with a prefix or suffix is not translated into a
1070  // header-name. This could theoretically be observable via the C++20
1071  // context-sensitive header-name formation rules.
1072  StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
1073  if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1074  FilenameTok.setKind(tok::header_name);
1075  }
1076 
1077  return false;
1078 }
1079 
1080 /// Collect the tokens of a C++20 pp-import-suffix.
1082  // FIXME: For error recovery, consider recognizing attribute syntax here
1083  // and terminating / diagnosing a missing semicolon if we find anything
1084  // else? (Can we leave that to the parser?)
1085  unsigned BracketDepth = 0;
1086  while (true) {
1087  Toks.emplace_back();
1088  Lex(Toks.back());
1089 
1090  switch (Toks.back().getKind()) {
1091  case tok::l_paren: case tok::l_square: case tok::l_brace:
1092  ++BracketDepth;
1093  break;
1094 
1095  case tok::r_paren: case tok::r_square: case tok::r_brace:
1096  if (BracketDepth == 0)
1097  return;
1098  --BracketDepth;
1099  break;
1100 
1101  case tok::semi:
1102  if (BracketDepth == 0)
1103  return;
1104  break;
1105 
1106  case tok::eof:
1107  return;
1108 
1109  default:
1110  break;
1111  }
1112  }
1113 }
1114 
1115 
1116 /// Lex a token following the 'import' contextual keyword.
1117 ///
1118 /// pp-import: [C++20]
1119 /// import header-name pp-import-suffix[opt] ;
1120 /// import header-name-tokens pp-import-suffix[opt] ;
1121 /// [ObjC] @ import module-name ;
1122 /// [Clang] import module-name ;
1123 ///
1124 /// header-name-tokens:
1125 /// string-literal
1126 /// < [any sequence of preprocessing-tokens other than >] >
1127 ///
1128 /// module-name:
1129 /// module-name-qualifier[opt] identifier
1130 ///
1131 /// module-name-qualifier
1132 /// module-name-qualifier[opt] identifier .
1133 ///
1134 /// We respond to a pp-import by importing macros from the named module.
1136  // Figure out what kind of lexer we actually have.
1138 
1139  // Lex the next token. The header-name lexing rules are used at the start of
1140  // a pp-import.
1141  //
1142  // For now, we only support header-name imports in C++20 mode.
1143  // FIXME: Should we allow this in all language modes that support an import
1144  // declaration as an extension?
1145  if (ModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1146  if (LexHeaderName(Result))
1147  return true;
1148  } else {
1149  Lex(Result);
1150  }
1151 
1152  // Allocate a holding buffer for a sequence of tokens and introduce it into
1153  // the token stream.
1154  auto EnterTokens = [this](ArrayRef<Token> Toks) {
1155  auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1156  std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1157  EnterTokenStream(std::move(ToksCopy), Toks.size(),
1158  /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1159  };
1160 
1161  // Check for a header-name.
1162  SmallVector<Token, 32> Suffix;
1163  if (Result.is(tok::header_name)) {
1164  // Enter the header-name token into the token stream; a Lex action cannot
1165  // both return a token and cache tokens (doing so would corrupt the token
1166  // cache if the call to Lex comes from CachingLex / PeekAhead).
1167  Suffix.push_back(Result);
1168 
1169  // Consume the pp-import-suffix and expand any macros in it now. We'll add
1170  // it back into the token stream later.
1171  CollectPpImportSuffix(Suffix);
1172  if (Suffix.back().isNot(tok::semi)) {
1173  // This is not a pp-import after all.
1174  EnterTokens(Suffix);
1175  return false;
1176  }
1177 
1178  // C++2a [cpp.module]p1:
1179  // The ';' preprocessing-token terminating a pp-import shall not have
1180  // been produced by macro replacement.
1181  SourceLocation SemiLoc = Suffix.back().getLocation();
1182  if (SemiLoc.isMacroID())
1183  Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1184 
1185  // Reconstitute the import token.
1186  Token ImportTok;
1187  ImportTok.startToken();
1188  ImportTok.setKind(tok::kw_import);
1189  ImportTok.setLocation(ModuleImportLoc);
1190  ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
1191  ImportTok.setLength(6);
1192 
1193  auto Action = HandleHeaderIncludeOrImport(
1194  /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
1195  switch (Action.Kind) {
1196  case ImportAction::None:
1197  break;
1198 
1199  case ImportAction::ModuleBegin:
1200  // Let the parser know we're textually entering the module.
1201  Suffix.emplace_back();
1202  Suffix.back().startToken();
1203  Suffix.back().setKind(tok::annot_module_begin);
1204  Suffix.back().setLocation(SemiLoc);
1205  Suffix.back().setAnnotationEndLoc(SemiLoc);
1206  Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1207  LLVM_FALLTHROUGH;
1208 
1209  case ImportAction::ModuleImport:
1210  case ImportAction::SkippedModuleImport:
1211  // We chose to import (or textually enter) the file. Convert the
1212  // header-name token into a header unit annotation token.
1213  Suffix[0].setKind(tok::annot_header_unit);
1214  Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1215  Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1216  // FIXME: Call the moduleImport callback?
1217  break;
1218  case ImportAction::Failure:
1219  assert(TheModuleLoader.HadFatalFailure &&
1220  "This should be an early exit only to a fatal error");
1221  Result.setKind(tok::eof);
1222  CurLexer->cutOffLexing();
1223  EnterTokens(Suffix);
1224  return true;
1225  }
1226 
1227  EnterTokens(Suffix);
1228  return false;
1229  }
1230 
1231  // The token sequence
1232  //
1233  // import identifier (. identifier)*
1234  //
1235  // indicates a module import directive. We already saw the 'import'
1236  // contextual keyword, so now we're looking for the identifiers.
1237  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1238  // We expected to see an identifier here, and we did; continue handling
1239  // identifiers.
1240  ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
1241  Result.getLocation()));
1242  ModuleImportExpectsIdentifier = false;
1243  CurLexerKind = CLK_LexAfterModuleImport;
1244  return true;
1245  }
1246 
1247  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1248  // see the next identifier. (We can also see a '[[' that begins an
1249  // attribute-specifier-seq here under the C++ Modules TS.)
1250  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1251  ModuleImportExpectsIdentifier = true;
1252  CurLexerKind = CLK_LexAfterModuleImport;
1253  return true;
1254  }
1255 
1256  // If we didn't recognize a module name at all, this is not a (valid) import.
1257  if (ModuleImportPath.empty() || Result.is(tok::eof))
1258  return true;
1259 
1260  // Consume the pp-import-suffix and expand any macros in it now, if we're not
1261  // at the semicolon already.
1262  SourceLocation SemiLoc = Result.getLocation();
1263  if (Result.isNot(tok::semi)) {
1264  Suffix.push_back(Result);
1265  CollectPpImportSuffix(Suffix);
1266  if (Suffix.back().isNot(tok::semi)) {
1267  // This is not an import after all.
1268  EnterTokens(Suffix);
1269  return false;
1270  }
1271  SemiLoc = Suffix.back().getLocation();
1272  }
1273 
1274  // Under the Modules TS, the dot is just part of the module name, and not
1275  // a real hierarchy separator. Flatten such module names now.
1276  //
1277  // FIXME: Is this the right level to be performing this transformation?
1278  std::string FlatModuleName;
1279  if (getLangOpts().ModulesTS || getLangOpts().CPlusPlusModules) {
1280  for (auto &Piece : ModuleImportPath) {
1281  if (!FlatModuleName.empty())
1282  FlatModuleName += ".";
1283  FlatModuleName += Piece.first->getName();
1284  }
1285  SourceLocation FirstPathLoc = ModuleImportPath[0].second;
1286  ModuleImportPath.clear();
1287  ModuleImportPath.push_back(
1288  std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1289  }
1290 
1291  Module *Imported = nullptr;
1292  if (getLangOpts().Modules) {
1293  Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1294  ModuleImportPath,
1296  /*IsInclusionDirective=*/false);
1297  if (Imported)
1298  makeModuleVisible(Imported, SemiLoc);
1299  }
1300  if (Callbacks)
1301  Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1302 
1303  if (!Suffix.empty()) {
1304  EnterTokens(Suffix);
1305  return false;
1306  }
1307  return true;
1308 }
1309 
1311  CurSubmoduleState->VisibleModules.setVisible(
1312  M, Loc, [](Module *) {},
1313  [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1314  // FIXME: Include the path in the diagnostic.
1315  // FIXME: Include the import location for the conflicting module.
1316  Diag(ModuleImportLoc, diag::warn_module_conflict)
1317  << Path[0]->getFullModuleName()
1318  << Conflict->getFullModuleName()
1319  << Message;
1320  });
1321 
1322  // Add this module to the imports list of the currently-built submodule.
1323  if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1324  BuildingSubmoduleStack.back().M->Imports.insert(M);
1325 }
1326 
1328  const char *DiagnosticTag,
1329  bool AllowMacroExpansion) {
1330  // We need at least one string literal.
1331  if (Result.isNot(tok::string_literal)) {
1332  Diag(Result, diag::err_expected_string_literal)
1333  << /*Source='in...'*/0 << DiagnosticTag;
1334  return false;
1335  }
1336 
1337  // Lex string literal tokens, optionally with macro expansion.
1338  SmallVector<Token, 4> StrToks;
1339  do {
1340  StrToks.push_back(Result);
1341 
1342  if (Result.hasUDSuffix())
1343  Diag(Result, diag::err_invalid_string_udl);
1344 
1345  if (AllowMacroExpansion)
1346  Lex(Result);
1347  else
1348  LexUnexpandedToken(Result);
1349  } while (Result.is(tok::string_literal));
1350 
1351  // Concatenate and parse the strings.
1352  StringLiteralParser Literal(StrToks, *this);
1353  assert(Literal.isAscii() && "Didn't allow wide strings in");
1354 
1355  if (Literal.hadError)
1356  return false;
1357 
1358  if (Literal.Pascal) {
1359  Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1360  << /*Source='in...'*/0 << DiagnosticTag;
1361  return false;
1362  }
1363 
1364  String = std::string(Literal.GetString());
1365  return true;
1366 }
1367 
1369  assert(Tok.is(tok::numeric_constant));
1370  SmallString<8> IntegerBuffer;
1371  bool NumberInvalid = false;
1372  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1373  if (NumberInvalid)
1374  return false;
1377  getDiagnostics());
1378  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1379  return false;
1380  llvm::APInt APVal(64, 0);
1381  if (Literal.GetIntegerValue(APVal))
1382  return false;
1383  Lex(Tok);
1384  Value = APVal.getLimitedValue();
1385  return true;
1386 }
1387 
1389  assert(Handler && "NULL comment handler");
1390  assert(!llvm::is_contained(CommentHandlers, Handler) &&
1391  "Comment handler already registered");
1392  CommentHandlers.push_back(Handler);
1393 }
1394 
1396  std::vector<CommentHandler *>::iterator Pos =
1397  llvm::find(CommentHandlers, Handler);
1398  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1399  CommentHandlers.erase(Pos);
1400 }
1401 
1403  bool AnyPendingTokens = false;
1404  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1405  HEnd = CommentHandlers.end();
1406  H != HEnd; ++H) {
1407  if ((*H)->HandleComment(*this, Comment))
1408  AnyPendingTokens = true;
1409  }
1410  if (!AnyPendingTokens || getCommentRetentionState())
1411  return false;
1412  Lex(result);
1413  return true;
1414 }
1415 
1416 void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1417  const MacroAnnotations &A =
1418  getMacroAnnotations(Identifier.getIdentifierInfo());
1419  assert(A.DeprecationInfo &&
1420  "Macro deprecation warning without recorded annotation!");
1421  const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1422  if (Info.Message.empty())
1423  Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1424  << Identifier.getIdentifierInfo() << 0;
1425  else
1426  Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1427  << Identifier.getIdentifierInfo() << 1 << Info.Message;
1428  Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1429 }
1430 
1431 void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1432  const MacroAnnotations &A =
1433  getMacroAnnotations(Identifier.getIdentifierInfo());
1434  assert(A.RestrictExpansionInfo &&
1435  "Macro restricted expansion warning without recorded annotation!");
1436  const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1437  if (Info.Message.empty())
1438  Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1439  << Identifier.getIdentifierInfo() << 0;
1440  else
1441  Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1442  << Identifier.getIdentifierInfo() << 1 << Info.Message;
1443  Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1444 }
1445 
1446 void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1447  bool IsUndef) const {
1448  const MacroAnnotations &A =
1449  getMacroAnnotations(Identifier.getIdentifierInfo());
1450  assert(A.FinalAnnotationLoc &&
1451  "Final macro warning without recorded annotation!");
1452 
1453  Diag(Identifier, diag::warn_pragma_final_macro)
1454  << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1455  Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1456 }
1457 
1458 ModuleLoader::~ModuleLoader() = default;
1459 
1461 
1463 
1465 
1467  if (Record)
1468  return;
1469 
1470  Record = new PreprocessingRecord(getSourceManager());
1471  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1472 }
clang::Token::startToken
void startToken()
Reset all flags to cleared.
Definition: Token.h:170
clang::MacroDirective::DefInfo::getLocation
SourceLocation getLocation() const
Definition: MacroInfo.h:486
clang::MacroInfo::getNumTokens
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:236
PreprocessorLexer.h
Builtins.h
clang::MacroInfo::getReplacementToken
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:238
clang::Preprocessor::CreateString
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
Definition: Preprocessor.cpp:486
clang::Token::IsReinjected
@ IsReinjected
Definition: Token.h:87
clang::Preprocessor::PrintStats
void PrintStats()
Definition: Preprocessor.cpp:273
clang::Preprocessor::FinalizeForModelFile
void FinalizeForModelFile()
Cleanup after model file parsing.
Definition: Preprocessor.cpp:228
clang::Token::hasUCN
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:295
clang::Token::hasLeadingSpace
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:271
clang::interp::APInt
llvm::APInt APInt
Definition: Integral.h:27
clang::Preprocessor::FinishLexStringLiteral
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
Definition: Preprocessor.cpp:1327
clang::Preprocessor::setCodeCompletionReached
void setCodeCompletionReached()
Note that we hit the code-completion point.
Definition: Preprocessor.h:1734
clang::Preprocessor::getMacroDefinition
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:1146
clang::Decl::getEndLoc
SourceLocation getEndLoc() const LLVM_READONLY
Definition: DeclBase.h:421
clang::DeclaratorContext::File
@ File
clang::Preprocessor::SetCodeCompletionPoint
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
Definition: Preprocessor.cpp:392
clang::SourceRange
A trivial tuple used to represent a source range.
Definition: SourceLocation.h:210
string
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD) string(SUBSTRING $
Definition: CMakeLists.txt:22
clang::Preprocessor::getLastMacroWithSpelling
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
Definition: Preprocessor.cpp:356
clang::Preprocessor::setCodeCompletionTokenRange
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Definition: Preprocessor.h:1359
clang::Preprocessor::Lex
void Lex(Token &Result)
Lex the next token for this preprocessor.
Definition: Preprocessor.cpp:892
clang::Preprocessor::HandleIdentifier
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Definition: Preprocessor.cpp:800
clang::MacroInfo::isObjectLike
bool isObjectLike() const
Definition: MacroInfo.h:203
clang::Preprocessor::macro_iterator
MacroMap::const_iterator macro_iterator
Definition: Preprocessor.h:1247
clang::MacroDefinition
A description of the current definition of a macro.
Definition: MacroInfo.h:588
clang::Preprocessor::LexHeaderName
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
Definition: Preprocessor.cpp:992
llvm::SmallVector
Definition: LLVM.h:38
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::Preprocessor::TUKind
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:291
clang::MacroInfo::tokens_begin
const_tokens_iterator tokens_begin() const
Definition: MacroInfo.h:245
clang::SourceLocation::getLocWithOffset
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
Definition: SourceLocation.h:134
clang::Preprocessor::getCommentRetentionState
bool getCommentRetentionState() const
Definition: Preprocessor.h:1039
TargetInfo.h
clang::MultiVersionKind::Target
@ Target
clang::Token::isExpandDisabled
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6....
Definition: Token.h:275
clang::PreprocessorLexer::setConditionalLevels
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
Definition: PreprocessorLexer.h:175
clang::Token::getIdentifierInfo
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:178
clang::MacroArgs
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
clang::DiagnosticsEngine
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
clang::Preprocessor::LookupFile
Optional< FileEntryRef > LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
Definition: PPDirectives.cpp:899
clang::TargetInfo
Exposes information about the current target.
Definition: TargetInfo.h:186
memcpy
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
Definition: __clang_cuda_device_functions.h:1549
clang::PreprocessingRecord
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
Definition: PreprocessingRecord.h:307
Module.h
clang::Token::isAtStartOfLine
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:267
clang::IdentifierInfo::isOutOfDate
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Definition: IdentifierTable.h:414
clang::Preprocessor::CodeCompleteIncludedFile
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
Definition: Preprocessor.cpp:448
llvm::Optional< llvm::MemoryBufferRef >
clang::Preprocessor::macro_end
macro_iterator macro_end(bool IncludeExternalMacros=true) const
Definition: Preprocessor.cpp:339
clang::ModuleLoader::HadFatalFailure
bool HadFatalFailure
Definition: ModuleLoader.h:157
clang::MacroInfo
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceManager.h
clang::Token::setIdentifierInfo
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:187
clang::MacroDirective::DefInfo::getMacroInfo
MacroInfo * getMacroInfo()
Definition: MacroInfo.h:492
Identifier
StringRef Identifier
Definition: Format.cpp:2559
clang::IdentifierInfo::getTokenID
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
Definition: IdentifierTable.h:261
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
LiteralSupport.h
clang::CommentHandler
Abstract base class that describes a handler that will receive source ranges for each of the comments...
Definition: Preprocessor.h:2610
End
SourceLocation End
Definition: USRLocFinder.cpp:167
TokenLexer.h
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:627
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:994
Preprocessor.h
clang::HeaderSearch
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:223
clang::ModuleLoader::loadModule
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
clang::Preprocessor::creatingPCHWithPragmaHdrStop
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Definition: Preprocessor.cpp:617
clang::TU_Prefix
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:822
clang::Preprocessor::usingPCHWithThroughHeader
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
Definition: Preprocessor.cpp:612
clang::PreprocessorLexer::LexIncludeFilename
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
Definition: PreprocessorLexer.cpp:32
clang::dataflow::Literal
uint32_t Literal
Literals are represented as positive integers.
Definition: WatchedLiteralsSolver.cpp:55
clang::CodeCompletionHandler::CodeCompleteIncludedFile
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an include directive.
Definition: CodeCompletionHandler.h:67
clang::SourceLocation::print
void print(raw_ostream &OS, const SourceManager &SM) const
Definition: SourceLocation.cpp:62
clang::Preprocessor::SetPoisonReason
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
Definition: Preprocessor.cpp:735
clang::Token::setRawIdentifierData
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:208
clang::Module
Describes a module or submodule.
Definition: Module.h:96
clang::Module::Hidden
@ Hidden
All of the names in this module are hidden.
Definition: Module.h:331
clang::Preprocessor::EnterSourceFile
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Definition: PPLexerChange.cpp:68
clang::Preprocessor::EndSourceFile
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
Definition: Preprocessor.cpp:687
clang::Decl::getKind
Kind getKind() const
Definition: DeclBase.h:428
clang::XRayInstrKind::None
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
Pragma.h
clang::Preprocessor::InitializeForModelFile
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
Definition: Preprocessor.cpp:216
clang::Preprocessor::addCommentHandler
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
Definition: Preprocessor.cpp:1388
clang::Token::setFlagValue
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:258
clang::Token::getKind
tok::TokenKind getKind() const
Definition: Token.h:92
clang::IdentifierInfo::setIsFutureCompatKeyword
void setIsFutureCompatKeyword(bool Val)
Definition: IdentifierTable.h:335
clang::ExternalPreprocessorSource::~ExternalPreprocessorSource
virtual ~ExternalPreprocessorSource()
clang::SourceManager::getMainFileID
FileID getMainFileID() const
Returns the FileID of the main source file.
Definition: SourceManager.h:841
clang::FileID::isValid
bool isValid() const
Definition: SourceLocation.h:44
clang::IdentifierInfo::setIsPoisoned
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
Definition: IdentifierTable.h:345
LangOptions.h
clang::Preprocessor::isPCHThroughHeader
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
Definition: Preprocessor.cpp:601
clang::Preprocessor::setTUFPEvalMethod
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2125
LexDiagnostic.h
clang::Preprocessor::Diag
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
Definition: Preprocessor.h:1806
ModuleLoader.h
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:97
clang::HeaderSearch::lookupModule
Module * lookupModule(StringRef ModuleName, SourceLocation ImportLoc=SourceLocation(), bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
Definition: HeaderSearch.cpp:278
clang::Preprocessor::getTargetInfo
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:995
clang::Preprocessor::createPreprocessingRecord
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
Definition: Preprocessor.cpp:1466
clang::Token::DisableExpand
@ DisableExpand
Definition: Token.h:77
IdentifierTable.h
clang::tok::getTokenName
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:24
llvm::SmallString
Definition: LLVM.h:37
clang::Token::needsCleaning
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:284
clang::Token::LeadingEmptyMacro
@ LeadingEmptyMacro
Definition: Token.h:79
clang::Preprocessor::LexAfterModuleImport
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
Definition: Preprocessor.cpp:1135
clang::CommentHandler::~CommentHandler
virtual ~CommentHandler()
clang::ExternalPreprocessorSource::ReadDefinedMacros
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
clang::SrcMgr::C_User
@ C_User
Definition: SourceManager.h:80
clang::Preprocessor::EnterMainSourceFile
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
Definition: Preprocessor.cpp:535
Line
const AnnotatedLine * Line
Definition: UsingDeclarationsSorter.cpp:68
clang::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
clang::FileEntry
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:330
clang::Preprocessor::SplitToken
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Definition: Preprocessor.cpp:506
clang::Token::isLiteral
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:114
clang::Token::getLength
unsigned getLength() const
Definition: Token.h:128
clang::SourceManager::isBeforeInTranslationUnit
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
Definition: SourceManager.cpp:2014
clang::Token::LeadingSpace
@ LeadingSpace
Definition: Token.h:75
clang::Preprocessor::getSourceManager
SourceManager & getSourceManager() const
Definition: Preprocessor.h:998
clang::Preprocessor::HandleComment
bool HandleComment(Token &result, SourceRange Comment)
Definition: Preprocessor.cpp:1402
clang::Preprocessor::getSpelling
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
Definition: Preprocessor.h:1821
MacroArgs.h
clang::Preprocessor::PoisonSEHIdentifiers
void PoisonSEHIdentifiers(bool Poison=true)
Definition: Preprocessor.cpp:739
clang::expandUCNs
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
Definition: LiteralSupport.cpp:327
clang::Preprocessor::CollectPpImportSuffix
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
Definition: Preprocessor.cpp:1081
FileSystemStatCache.h
SourceLocation.h
clang::Preprocessor::SkipTokensWhileUsingPCH
void SkipTokensWhileUsingPCH()
Skip tokens until after the include of the through header or until after a #pragma hdrstop.
Definition: Preprocessor.cpp:630
clang::Preprocessor::getTotalMemory
size_t getTotalMemory() const
Definition: Preprocessor.cpp:326
clang::Preprocessor::CodeCompleteNaturalLanguage
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Definition: Preprocessor.cpp:455
clang::Token::setLiteralData
void setLiteralData(const char *Ptr)
Definition: Token.h:220
clang::SourceManager::getMemoryBufferForFileOrNone
llvm::Optional< llvm::MemoryBufferRef > getMemoryBufferForFileOrNone(const FileEntry *File)
Retrieve the memory buffer associated with the given file.
Definition: SourceManager.cpp:682
clang::ScratchBuffer
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Definition: ScratchBuffer.h:24
clang::Preprocessor::DumpLocation
void DumpLocation(SourceLocation Loc) const
Definition: Preprocessor.cpp:260
MacroDefinitionEquals
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
Definition: Preprocessor.cpp:350
getFutureCompatDiagKind
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
Definition: Preprocessor.cpp:766
clang::Preprocessor::usingPCHWithPragmaHdrStop
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
Definition: Preprocessor.cpp:621
clang::IdentifierInfo::isPoisoned
bool isPoisoned() const
Return true if this token has been poisoned.
Definition: IdentifierTable.h:354
PreprocessingRecord.h
llvm::ArrayRef
Definition: LLVM.h:34
Lexer.h
Value
Value
Definition: UninitializedValues.cpp:102
clang::Preprocessor::setCodeCompletionIdentifierInfo
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
Definition: Preprocessor.h:1353
clang::SourceManager::getCharacterData
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer.
Definition: SourceManager.cpp:1152
HeaderSearch.h
LLVM.h
clang::Preprocessor::DumpToken
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
Definition: Preprocessor.cpp:234
clang::Token::StartOfLine
@ StartOfLine
Definition: Token.h:73
clang::NumericLiteralParser
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber,...
Definition: LiteralSupport.h:42
clang::Preprocessor::getCurrentModule
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
Definition: Preprocessor.cpp:522
clang::ModuleLoader
Abstract interface for a module loader.
Definition: ModuleLoader.h:77
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:84
clang::SourceManager::overrideFileContents
void overrideFileContents(const FileEntry *SourceFile, const llvm::MemoryBufferRef &Buffer)
Override the contents of the given source file by providing an already-allocated buffer.
Definition: SourceManager.h:952
clang::IdentifierTable::AddKeywords
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
Definition: IdentifierTable.cpp:213
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:125
clang::SourceLocation::isMacroID
bool isMacroID() const
Definition: SourceLocation.h:103
clang::LangOptions
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:78
CodeCompletionHandler.h
clang::Token::isAnnotation
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:119
clang::Preprocessor::removeCommentHandler
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Definition: Preprocessor.cpp:1395
clang::SourceManager::isLoadedFileID
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
Definition: SourceManager.h:1740
clang::Token::setLocation
void setLocation(SourceLocation L)
Definition: Token.h:133
std
Definition: Format.h:4296
clang::TranslationUnitKind
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:816
clang::IdentifierInfo::getName
StringRef getName() const
Return the actual identifier string.
Definition: IdentifierTable.h:195
clang::SourceLocation::isInvalid
bool isInvalid() const
Definition: SourceLocation.h:111
clang
Definition: CalledOnceCheck.h:17
clang::Preprocessor::addPPCallbacks
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:1092
clang::SourceManager::createFileID
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
Definition: SourceManager.cpp:532
clang::ExternalPreprocessorSource::updateOutOfDateIdentifier
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
clang::Preprocessor::markIncluded
bool markIncluded(const FileEntry *File)
Mark the file as included.
Definition: Preprocessor.h:1263
clang::CodeCompletionHandler::CodeCompleteNaturalLanguage
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Definition: CodeCompletionHandler.h:72
clang::Preprocessor::creatingPCHWithThroughHeader
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
Definition: Preprocessor.cpp:607
clang::Preprocessor::LookUpIdentifierInfo
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Definition: Preprocessor.cpp:700
clang::PragmaHandlerRegistry
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Definition: Preprocessor.h:2630
MacroInfo.h
clang::Preprocessor::getDiagnostics
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:991
clang::Preprocessor::Preprocessor
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
Definition: Preprocessor.cpp:79
clang::EmptylineHandler::~EmptylineHandler
virtual ~EmptylineHandler()
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:38
clang::SourceLocation::isValid
bool isValid() const
Return true if this is a valid SourceLocation object.
Definition: SourceLocation.h:110
clang::Token::setLength
void setLength(unsigned Len)
Definition: Token.h:134
clang::FileID::isInvalid
bool isInvalid() const
Definition: SourceLocation.h:45
clang::SourceManager::getFileEntryForID
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
Definition: SourceManager.h:1035
clang::Token::setKind
void setKind(tok::TokenKind K)
Definition: Token.h:93
clang::IdentifierInfoLookup
Provides lookups to, and iteration over, IdentiferInfo objects.
Definition: IdentifierTable.h:534
clang::Preprocessor::makeModuleVisible
void makeModuleVisible(Module *M, SourceLocation Loc)
Definition: Preprocessor.cpp:1310
clang::Preprocessor::recomputeCurLexerKind
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Definition: Preprocessor.cpp:383
clang::Preprocessor::DumpMacro
void DumpMacro(const MacroInfo &MI) const
Definition: Preprocessor.cpp:264
clang::IdentifierInfo::isModulesImport
bool isModulesImport() const
Determine whether this is the contextual keyword import.
Definition: IdentifierTable.h:427
clang::IdentifierInfo::isExtensionToken
bool isExtensionToken() const
get/setExtension - Initialize information about whether or not this language token is an extension.
Definition: IdentifierTable.h:320
clang::Token::isOneOf
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
clang::IdentifierInfo::isFutureCompatKeyword
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Definition: IdentifierTable.h:334
ScratchBuffer.h
clang::CodeCompletionHandler::~CodeCompletionHandler
virtual ~CodeCompletionHandler()
clang::comments::tok::eof
@ eof
Definition: CommentLexer.h:33
llvm::SmallVectorImpl< char >
FileManager.h
clang::PragmaNamespace
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:96
clang::Preprocessor::LexUnexpandedToken
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Definition: Preprocessor.h:1539
true
#define true
Definition: stdbool.h:21
PreprocessorOptions.h
SM
#define SM(sm)
Definition: Cuda.cpp:81
clang::Preprocessor::HandlePoisonedIdentifier
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
Definition: Preprocessor.cpp:753
clang::Preprocessor::getHeaderSearchInfo
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:999
Token.h
clang::MacroDirective::DefInfo
Definition: MacroInfo.h:362
clang::StringLiteralParser
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
Definition: LiteralSupport.h:218
clang::Preprocessor::getExternalSource
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:1011
clang::Decl::getLocation
SourceLocation getLocation() const
Definition: DeclBase.h:425
clang::Preprocessor::getMacroAnnotations
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
Definition: Preprocessor.h:2577
clang::Preprocessor::~Preprocessor
~Preprocessor()
Definition: Preprocessor.cpp:169
clang::Preprocessor::getPredefinesFileID
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:1084
clang::diag::kind
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Definition: DiagnosticIDs.h:62
clang::HeaderSearch::setTarget
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Definition: HeaderSearch.cpp:721
clang::SourceManager::createExpansionLoc
SourceLocation createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned Length, bool ExpansionIsTokenRange=true, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Creates an expansion SLocEntry for a macro use.
Definition: SourceManager.cpp:639
clang::ModuleMacro
Represents a macro directive exported by a module.
Definition: MacroInfo.h:512
clang::Preprocessor::getIdentifierInfo
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Definition: Preprocessor.h:1290
clang::Preprocessor::macro_begin
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Definition: Preprocessor.cpp:312
clang::Preprocessor::parseSimpleIntegerLiteral
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
Definition: Preprocessor.cpp:1368
ExternalPreprocessorSource.h
clang::Preprocessor::Initialize
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Definition: Preprocessor.cpp:195
clang::ModuleLoader::~ModuleLoader
virtual ~ModuleLoader()
clang::Token::hasLeadingEmptyMacro
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
Definition: Token.h:288