clang  17.0.0git
Preprocessor.cpp
Go to the documentation of this file.
1 //===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Preprocessor interface.
10 //
11 //===----------------------------------------------------------------------===//
12 //
13 // Options to support:
14 // -H - Print the name of each header file used.
15 // -d[DNI] - Dump various things.
16 // -fworking-directory - #line's with preprocessor's working dir.
17 // -fpreprocessed
18 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19 // -W*
20 // -w
21 //
22 // Messages to emit:
23 // "Multiple include guards may be useful for:\n"
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "clang/Lex/Preprocessor.h"
28 #include "clang/Basic/Builtins.h"
32 #include "clang/Basic/LLVM.h"
34 #include "clang/Basic/Module.h"
37 #include "clang/Basic/TargetInfo.h"
40 #include "clang/Lex/HeaderSearch.h"
42 #include "clang/Lex/Lexer.h"
44 #include "clang/Lex/MacroArgs.h"
45 #include "clang/Lex/MacroInfo.h"
46 #include "clang/Lex/ModuleLoader.h"
47 #include "clang/Lex/Pragma.h"
52 #include "clang/Lex/Token.h"
53 #include "clang/Lex/TokenLexer.h"
54 #include "llvm/ADT/APInt.h"
55 #include "llvm/ADT/ArrayRef.h"
56 #include "llvm/ADT/DenseMap.h"
57 #include "llvm/ADT/STLExtras.h"
58 #include "llvm/ADT/SmallString.h"
59 #include "llvm/ADT/SmallVector.h"
60 #include "llvm/ADT/StringRef.h"
61 #include "llvm/Support/Capacity.h"
62 #include "llvm/Support/ErrorHandling.h"
63 #include "llvm/Support/MemoryBuffer.h"
64 #include "llvm/Support/raw_ostream.h"
65 #include <algorithm>
66 #include <cassert>
67 #include <memory>
68 #include <optional>
69 #include <string>
70 #include <utility>
71 #include <vector>
72 
73 using namespace clang;
74 
75 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
76 
78 
79 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
80  DiagnosticsEngine &diags, LangOptions &opts,
81  SourceManager &SM, HeaderSearch &Headers,
82  ModuleLoader &TheModuleLoader,
83  IdentifierInfoLookup *IILookup, bool OwnsHeaders,
84  TranslationUnitKind TUKind)
85  : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86  FileMgr(Headers.getFileMgr()), SourceMgr(SM),
87  ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88  TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
89  // As the language options may have not been loaded yet (when
90  // deserializing an ASTUnit), adding keywords to the identifier table is
91  // deferred to Preprocessor::Initialize().
92  Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
93  TUKind(TUKind), SkipMainFilePreamble(0, true),
94  CurSubmoduleState(&NullSubmoduleState) {
95  OwnsHeaderSearch = OwnsHeaders;
96 
97  // Default to discarding comments.
98  KeepComments = false;
99  KeepMacroComments = false;
100  SuppressIncludeNotFoundError = false;
101 
102  // Macro expansion is enabled.
103  DisableMacroExpansion = false;
104  MacroExpansionInDirectivesOverride = false;
105  InMacroArgs = false;
106  ArgMacro = nullptr;
107  InMacroArgPreExpansion = false;
108  NumCachedTokenLexers = 0;
109  PragmasEnabled = true;
110  ParsingIfOrElifDirective = false;
111  PreprocessedOutput = false;
112 
113  // We haven't read anything from the external source.
114  ReadMacrosFromExternalSource = false;
115 
116  BuiltinInfo = std::make_unique<Builtin::Context>();
117 
118  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
119  // a macro. They get unpoisoned where it is allowed.
120  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
121  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
122  (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
123  SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
124 
125  // Initialize the pragma handlers.
126  RegisterBuiltinPragmas();
127 
128  // Initialize builtin macros like __LINE__ and friends.
129  RegisterBuiltinMacros();
130 
131  if(LangOpts.Borland) {
132  Ident__exception_info = getIdentifierInfo("_exception_info");
133  Ident___exception_info = getIdentifierInfo("__exception_info");
134  Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
135  Ident__exception_code = getIdentifierInfo("_exception_code");
136  Ident___exception_code = getIdentifierInfo("__exception_code");
137  Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
138  Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
139  Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
140  Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
141  } else {
142  Ident__exception_info = Ident__exception_code = nullptr;
143  Ident__abnormal_termination = Ident___exception_info = nullptr;
144  Ident___exception_code = Ident___abnormal_termination = nullptr;
145  Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
146  Ident_AbnormalTermination = nullptr;
147  }
148 
149  // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
151  SkippingUntilPragmaHdrStop = true;
152 
153  // If using a PCH with a through header, start skipping tokens.
154  if (!this->PPOpts->PCHThroughHeader.empty() &&
155  !this->PPOpts->ImplicitPCHInclude.empty())
156  SkippingUntilPCHThroughHeader = true;
157 
158  if (this->PPOpts->GeneratePreamble)
159  PreambleConditionalStack.startRecording();
160 
161  MaxTokens = LangOpts.MaxTokens;
162 }
163 
165  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
166 
167  IncludeMacroStack.clear();
168 
169  // Free any cached macro expanders.
170  // This populates MacroArgCache, so all TokenLexers need to be destroyed
171  // before the code below that frees up the MacroArgCache list.
172  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
173  CurTokenLexer.reset();
174 
175  // Free any cached MacroArgs.
176  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
177  ArgList = ArgList->deallocate();
178 
179  // Delete the header search info, if we own it.
180  if (OwnsHeaderSearch)
181  delete &HeaderInfo;
182 }
183 
185  const TargetInfo *AuxTarget) {
186  assert((!this->Target || this->Target == &Target) &&
187  "Invalid override of target information");
188  this->Target = &Target;
189 
190  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
191  "Invalid override of aux target information.");
192  this->AuxTarget = AuxTarget;
193 
194  // Initialize information about built-ins.
195  BuiltinInfo->InitializeTarget(Target, AuxTarget);
196  HeaderInfo.setTarget(Target);
197 
198  // Populate the identifier table with info about keywords for the current language.
199  Identifiers.AddKeywords(LangOpts);
200 
201  // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
202  setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
203 
204  if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine)
205  // Use setting from TargetInfo.
206  setCurrentFPEvalMethod(SourceLocation(), Target.getFPEvalMethod());
207  else
208  // Set initial value of __FLT_EVAL_METHOD__ from the command line.
209  setCurrentFPEvalMethod(SourceLocation(), getLangOpts().getFPEvalMethod());
210  // When `-ffast-math` option is enabled, it triggers several driver math
211  // options to be enabled. Among those, only one the following two modes
212  // affect the eval-method: reciprocal or reassociate.
213  if (getLangOpts().AllowFPReassoc || getLangOpts().AllowRecip)
215 }
216 
218  NumEnteredSourceFiles = 0;
219 
220  // Reset pragmas
221  PragmaHandlersBackup = std::move(PragmaHandlers);
222  PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
223  RegisterBuiltinPragmas();
224 
225  // Reset PredefinesFileID
226  PredefinesFileID = FileID();
227 }
228 
230  NumEnteredSourceFiles = 1;
231 
232  PragmaHandlers = std::move(PragmaHandlersBackup);
233 }
234 
235 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
236  llvm::errs() << tok::getTokenName(Tok.getKind());
237 
238  if (!Tok.isAnnotation())
239  llvm::errs() << " '" << getSpelling(Tok) << "'";
240 
241  if (!DumpFlags) return;
242 
243  llvm::errs() << "\t";
244  if (Tok.isAtStartOfLine())
245  llvm::errs() << " [StartOfLine]";
246  if (Tok.hasLeadingSpace())
247  llvm::errs() << " [LeadingSpace]";
248  if (Tok.isExpandDisabled())
249  llvm::errs() << " [ExpandDisabled]";
250  if (Tok.needsCleaning()) {
251  const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
252  llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
253  << "']";
254  }
255 
256  llvm::errs() << "\tLoc=<";
257  DumpLocation(Tok.getLocation());
258  llvm::errs() << ">";
259 }
260 
262  Loc.print(llvm::errs(), SourceMgr);
263 }
264 
265 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
266  llvm::errs() << "MACRO: ";
267  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
269  llvm::errs() << " ";
270  }
271  llvm::errs() << "\n";
272 }
273 
275  llvm::errs() << "\n*** Preprocessor Stats:\n";
276  llvm::errs() << NumDirectives << " directives found:\n";
277  llvm::errs() << " " << NumDefined << " #define.\n";
278  llvm::errs() << " " << NumUndefined << " #undef.\n";
279  llvm::errs() << " #include/#include_next/#import:\n";
280  llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
281  llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
282  llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
283  llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
284  llvm::errs() << " " << NumEndif << " #endif.\n";
285  llvm::errs() << " " << NumPragma << " #pragma.\n";
286  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
287 
288  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
289  << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
290  << NumFastMacroExpanded << " on the fast path.\n";
291  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
292  << " token paste (##) operations performed, "
293  << NumFastTokenPaste << " on the fast path.\n";
294 
295  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
296 
297  llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
298  llvm::errs() << "\n Macro Expanded Tokens: "
299  << llvm::capacity_in_bytes(MacroExpandedTokens);
300  llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
301  // FIXME: List information for all submodules.
302  llvm::errs() << "\n Macros: "
303  << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
304  llvm::errs() << "\n #pragma push_macro Info: "
305  << llvm::capacity_in_bytes(PragmaPushMacroInfo);
306  llvm::errs() << "\n Poison Reasons: "
307  << llvm::capacity_in_bytes(PoisonReasons);
308  llvm::errs() << "\n Comment Handlers: "
309  << llvm::capacity_in_bytes(CommentHandlers) << "\n";
310 }
311 
313 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
314  if (IncludeExternalMacros && ExternalSource &&
315  !ReadMacrosFromExternalSource) {
316  ReadMacrosFromExternalSource = true;
317  ExternalSource->ReadDefinedMacros();
318  }
319 
320  // Make sure we cover all macros in visible modules.
321  for (const ModuleMacro &Macro : ModuleMacros)
322  CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
323 
324  return CurSubmoduleState->Macros.begin();
325 }
326 
328  return BP.getTotalMemory()
329  + llvm::capacity_in_bytes(MacroExpandedTokens)
330  + Predefines.capacity() /* Predefines buffer. */
331  // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
332  // and ModuleMacros.
333  + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
334  + llvm::capacity_in_bytes(PragmaPushMacroInfo)
335  + llvm::capacity_in_bytes(PoisonReasons)
336  + llvm::capacity_in_bytes(CommentHandlers);
337 }
338 
340 Preprocessor::macro_end(bool IncludeExternalMacros) const {
341  if (IncludeExternalMacros && ExternalSource &&
342  !ReadMacrosFromExternalSource) {
343  ReadMacrosFromExternalSource = true;
344  ExternalSource->ReadDefinedMacros();
345  }
346 
347  return CurSubmoduleState->Macros.end();
348 }
349 
350 /// Compares macro tokens with a specified token value sequence.
351 static bool MacroDefinitionEquals(const MacroInfo *MI,
352  ArrayRef<TokenValue> Tokens) {
353  return Tokens.size() == MI->getNumTokens() &&
354  std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
355 }
356 
358  SourceLocation Loc,
359  ArrayRef<TokenValue> Tokens) const {
360  SourceLocation BestLocation;
361  StringRef BestSpelling;
363  I != E; ++I) {
365  Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
366  if (!Def || !Def.getMacroInfo())
367  continue;
368  if (!Def.getMacroInfo()->isObjectLike())
369  continue;
370  if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
371  continue;
372  SourceLocation Location = Def.getLocation();
373  // Choose the macro defined latest.
374  if (BestLocation.isInvalid() ||
375  (Location.isValid() &&
376  SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
377  BestLocation = Location;
378  BestSpelling = I->first->getName();
379  }
380  }
381  return BestSpelling;
382 }
383 
385  if (CurLexer)
386  CurLexerKind = CurLexer->isDependencyDirectivesLexer()
387  ? CLK_DependencyDirectivesLexer
388  : CLK_Lexer;
389  else if (CurTokenLexer)
390  CurLexerKind = CLK_TokenLexer;
391  else
392  CurLexerKind = CLK_CachingLexer;
393 }
394 
396  unsigned CompleteLine,
397  unsigned CompleteColumn) {
398  assert(File);
399  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
400  assert(!CodeCompletionFile && "Already set");
401 
402  // Load the actual file's contents.
403  std::optional<llvm::MemoryBufferRef> Buffer =
405  if (!Buffer)
406  return true;
407 
408  // Find the byte position of the truncation point.
409  const char *Position = Buffer->getBufferStart();
410  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
411  for (; *Position; ++Position) {
412  if (*Position != '\r' && *Position != '\n')
413  continue;
414 
415  // Eat \r\n or \n\r as a single line.
416  if ((Position[1] == '\r' || Position[1] == '\n') &&
417  Position[0] != Position[1])
418  ++Position;
419  ++Position;
420  break;
421  }
422  }
423 
424  Position += CompleteColumn - 1;
425 
426  // If pointing inside the preamble, adjust the position at the beginning of
427  // the file after the preamble.
428  if (SkipMainFilePreamble.first &&
429  SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
430  if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
431  Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
432  }
433 
434  if (Position > Buffer->getBufferEnd())
435  Position = Buffer->getBufferEnd();
436 
437  CodeCompletionFile = File;
438  CodeCompletionOffset = Position - Buffer->getBufferStart();
439 
440  auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
441  Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
442  char *NewBuf = NewBuffer->getBufferStart();
443  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
444  *NewPos = '\0';
445  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
446  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
447 
448  return false;
449 }
450 
451 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
452  bool IsAngled) {
454  if (CodeComplete)
455  CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
456 }
457 
460  if (CodeComplete)
461  CodeComplete->CodeCompleteNaturalLanguage();
462 }
463 
464 /// getSpelling - This method is used to get the spelling of a token into a
465 /// SmallVector. Note that the returned StringRef may not point to the
466 /// supplied buffer if a copy can be avoided.
467 StringRef Preprocessor::getSpelling(const Token &Tok,
468  SmallVectorImpl<char> &Buffer,
469  bool *Invalid) const {
470  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
471  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
472  // Try the fast path.
473  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
474  return II->getName();
475  }
476 
477  // Resize the buffer if we need to copy into it.
478  if (Tok.needsCleaning())
479  Buffer.resize(Tok.getLength());
480 
481  const char *Ptr = Buffer.data();
482  unsigned Len = getSpelling(Tok, Ptr, Invalid);
483  return StringRef(Ptr, Len);
484 }
485 
486 /// CreateString - Plop the specified string into a scratch buffer and return a
487 /// location for it. If specified, the source location provides a source
488 /// location for the token.
489 void Preprocessor::CreateString(StringRef Str, Token &Tok,
490  SourceLocation ExpansionLocStart,
491  SourceLocation ExpansionLocEnd) {
492  Tok.setLength(Str.size());
493 
494  const char *DestPtr;
495  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
496 
497  if (ExpansionLocStart.isValid())
498  Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
499  ExpansionLocEnd, Str.size());
500  Tok.setLocation(Loc);
501 
502  // If this is a raw identifier or a literal token, set the pointer data.
503  if (Tok.is(tok::raw_identifier))
504  Tok.setRawIdentifierData(DestPtr);
505  else if (Tok.isLiteral())
506  Tok.setLiteralData(DestPtr);
507 }
508 
510  auto &SM = getSourceManager();
511  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
512  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
513  bool Invalid = false;
514  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
515  if (Invalid)
516  return SourceLocation();
517 
518  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
519  const char *DestPtr;
520  SourceLocation Spelling =
521  ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
522  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
523 }
524 
526  if (!getLangOpts().isCompilingModule())
527  return nullptr;
528 
529  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
530 }
531 
533  if (!getLangOpts().isCompilingModuleImplementation())
534  return nullptr;
535 
536  return getHeaderSearchInfo().lookupModule(getLangOpts().ModuleName);
537 }
538 
539 //===----------------------------------------------------------------------===//
540 // Preprocessor Initialization Methods
541 //===----------------------------------------------------------------------===//
542 
543 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
544 /// which implicitly adds the builtin defines etc.
546  // We do not allow the preprocessor to reenter the main file. Doing so will
547  // cause FileID's to accumulate information from both runs (e.g. #line
548  // information) and predefined macros aren't guaranteed to be set properly.
549  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
550  FileID MainFileID = SourceMgr.getMainFileID();
551 
552  // If MainFileID is loaded it means we loaded an AST file, no need to enter
553  // a main file.
554  if (!SourceMgr.isLoadedFileID(MainFileID)) {
555  // Enter the main file source buffer.
556  EnterSourceFile(MainFileID, nullptr, SourceLocation());
557 
558  // If we've been asked to skip bytes in the main file (e.g., as part of a
559  // precompiled preamble), do so now.
560  if (SkipMainFilePreamble.first > 0)
561  CurLexer->SetByteOffset(SkipMainFilePreamble.first,
562  SkipMainFilePreamble.second);
563 
564  // Tell the header info that the main file was entered. If the file is later
565  // #imported, it won't be re-entered.
566  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
567  markIncluded(FE);
568  }
569 
570  // Preprocess Predefines to populate the initial preprocessor state.
571  std::unique_ptr<llvm::MemoryBuffer> SB =
572  llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
573  assert(SB && "Cannot create predefined source buffer");
574  FileID FID = SourceMgr.createFileID(std::move(SB));
575  assert(FID.isValid() && "Could not create FileID for predefines?");
576  setPredefinesFileID(FID);
577 
578  // Start parsing the predefines.
579  EnterSourceFile(FID, nullptr, SourceLocation());
580 
581  if (!PPOpts->PCHThroughHeader.empty()) {
582  // Lookup and save the FileID for the through header. If it isn't found
583  // in the search path, it's a fatal error.
585  SourceLocation(), PPOpts->PCHThroughHeader,
586  /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
587  /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
588  /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
589  /*IsFrameworkFound=*/nullptr);
590  if (!File) {
591  Diag(SourceLocation(), diag::err_pp_through_header_not_found)
592  << PPOpts->PCHThroughHeader;
593  return;
594  }
595  setPCHThroughHeaderFileID(
597  }
598 
599  // Skip tokens from the Predefines and if needed the main file.
600  if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
601  (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
603 }
604 
605 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
606  assert(PCHThroughHeaderFileID.isInvalid() &&
607  "PCHThroughHeaderFileID already set!");
608  PCHThroughHeaderFileID = FID;
609 }
610 
612  assert(PCHThroughHeaderFileID.isValid() &&
613  "Invalid PCH through header FileID");
614  return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
615 }
616 
618  return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
619  PCHThroughHeaderFileID.isValid();
620 }
621 
623  return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
624  PCHThroughHeaderFileID.isValid();
625 }
626 
628  return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
629 }
630 
632  return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
633 }
634 
635 /// Skip tokens until after the #include of the through header or
636 /// until after a #pragma hdrstop is seen. Tokens in the predefines file
637 /// and the main file may be skipped. If the end of the predefines file
638 /// is reached, skipping continues into the main file. If the end of the
639 /// main file is reached, it's a fatal error.
641  bool ReachedMainFileEOF = false;
642  bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
643  bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
644  Token Tok;
645  while (true) {
646  bool InPredefines =
647  (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
648  switch (CurLexerKind) {
649  case CLK_Lexer:
650  CurLexer->Lex(Tok);
651  break;
652  case CLK_TokenLexer:
653  CurTokenLexer->Lex(Tok);
654  break;
655  case CLK_CachingLexer:
656  CachingLex(Tok);
657  break;
658  case CLK_DependencyDirectivesLexer:
659  CurLexer->LexDependencyDirectiveToken(Tok);
660  break;
661  case CLK_LexAfterModuleImport:
663  break;
664  }
665  if (Tok.is(tok::eof) && !InPredefines) {
666  ReachedMainFileEOF = true;
667  break;
668  }
669  if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
670  break;
671  if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
672  break;
673  }
674  if (ReachedMainFileEOF) {
675  if (UsingPCHThroughHeader)
676  Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
677  << PPOpts->PCHThroughHeader << 1;
678  else if (!PPOpts->PCHWithHdrStopCreate)
679  Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
680  }
681 }
682 
683 void Preprocessor::replayPreambleConditionalStack() {
684  // Restore the conditional stack from the preamble, if there is one.
685  if (PreambleConditionalStack.isReplaying()) {
686  assert(CurPPLexer &&
687  "CurPPLexer is null when calling replayPreambleConditionalStack.");
688  CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
689  PreambleConditionalStack.doneReplaying();
690  if (PreambleConditionalStack.reachedEOFWhileSkipping())
691  SkipExcludedConditionalBlock(
692  PreambleConditionalStack.SkipInfo->HashTokenLoc,
693  PreambleConditionalStack.SkipInfo->IfTokenLoc,
694  PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
695  PreambleConditionalStack.SkipInfo->FoundElse,
696  PreambleConditionalStack.SkipInfo->ElseLoc);
697  }
698 }
699 
701  // Notify the client that we reached the end of the source file.
702  if (Callbacks)
703  Callbacks->EndOfMainFile();
704 }
705 
706 //===----------------------------------------------------------------------===//
707 // Lexer Event Handling.
708 //===----------------------------------------------------------------------===//
709 
710 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
711 /// identifier information for the token and install it into the token,
712 /// updating the token kind accordingly.
714  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
715 
716  // Look up this token, see if it is a macro, or if it is a language keyword.
717  IdentifierInfo *II;
718  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
719  // No cleaning needed, just use the characters from the lexed buffer.
720  II = getIdentifierInfo(Identifier.getRawIdentifier());
721  } else {
722  // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
723  SmallString<64> IdentifierBuffer;
724  StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
725 
726  if (Identifier.hasUCN()) {
727  SmallString<64> UCNIdentifierBuffer;
728  expandUCNs(UCNIdentifierBuffer, CleanedStr);
729  II = getIdentifierInfo(UCNIdentifierBuffer);
730  } else {
731  II = getIdentifierInfo(CleanedStr);
732  }
733  }
734 
735  // Update the token info (identifier info and appropriate token kind).
736  // FIXME: the raw_identifier may contain leading whitespace which is removed
737  // from the cleaned identifier token. The SourceLocation should be updated to
738  // refer to the non-whitespace character. For instance, the text "\\\nB" (a
739  // line continuation before 'B') is parsed as a single tok::raw_identifier and
740  // is cleaned to tok::identifier "B". After cleaning the token's length is
741  // still 3 and the SourceLocation refers to the location of the backslash.
742  Identifier.setIdentifierInfo(II);
743  Identifier.setKind(II->getTokenID());
744 
745  return II;
746 }
747 
748 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
749  PoisonReasons[II] = DiagID;
750 }
751 
753  assert(Ident__exception_code && Ident__exception_info);
754  assert(Ident___exception_code && Ident___exception_info);
755  Ident__exception_code->setIsPoisoned(Poison);
756  Ident___exception_code->setIsPoisoned(Poison);
757  Ident_GetExceptionCode->setIsPoisoned(Poison);
758  Ident__exception_info->setIsPoisoned(Poison);
759  Ident___exception_info->setIsPoisoned(Poison);
760  Ident_GetExceptionInfo->setIsPoisoned(Poison);
761  Ident__abnormal_termination->setIsPoisoned(Poison);
762  Ident___abnormal_termination->setIsPoisoned(Poison);
763  Ident_AbnormalTermination->setIsPoisoned(Poison);
764 }
765 
767  assert(Identifier.getIdentifierInfo() &&
768  "Can't handle identifiers without identifier info!");
769  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
770  PoisonReasons.find(Identifier.getIdentifierInfo());
771  if(it == PoisonReasons.end())
772  Diag(Identifier, diag::err_pp_used_poisoned_id);
773  else
774  Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
775 }
776 
777 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
778  assert(II.isOutOfDate() && "not out of date");
780 }
781 
782 /// HandleIdentifier - This callback is invoked when the lexer reads an
783 /// identifier. This callback looks up the identifier in the map and/or
784 /// potentially macro expands it or turns it into a named token (like 'for').
785 ///
786 /// Note that callers of this method are guarded by checking the
787 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
788 /// IdentifierInfo methods that compute these properties will need to change to
789 /// match.
791  assert(Identifier.getIdentifierInfo() &&
792  "Can't handle identifiers without identifier info!");
793 
794  IdentifierInfo &II = *Identifier.getIdentifierInfo();
795 
796  // If the information about this identifier is out of date, update it from
797  // the external source.
798  // We have to treat __VA_ARGS__ in a special way, since it gets
799  // serialized with isPoisoned = true, but our preprocessor may have
800  // unpoisoned it if we're defining a C99 macro.
801  if (II.isOutOfDate()) {
802  bool CurrentIsPoisoned = false;
803  const bool IsSpecialVariadicMacro =
804  &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
805  if (IsSpecialVariadicMacro)
806  CurrentIsPoisoned = II.isPoisoned();
807 
808  updateOutOfDateIdentifier(II);
809  Identifier.setKind(II.getTokenID());
810 
811  if (IsSpecialVariadicMacro)
812  II.setIsPoisoned(CurrentIsPoisoned);
813  }
814 
815  // If this identifier was poisoned, and if it was not produced from a macro
816  // expansion, emit an error.
817  if (II.isPoisoned() && CurPPLexer) {
819  }
820 
821  // If this is a macro to be expanded, do it.
822  if (MacroDefinition MD = getMacroDefinition(&II)) {
823  auto *MI = MD.getMacroInfo();
824  assert(MI && "macro definition with no macro info?");
825  if (!DisableMacroExpansion) {
826  if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
827  // C99 6.10.3p10: If the preprocessing token immediately after the
828  // macro name isn't a '(', this macro should not be expanded.
829  if (!MI->isFunctionLike() || isNextPPTokenLParen())
830  return HandleMacroExpandedIdentifier(Identifier, MD);
831  } else {
832  // C99 6.10.3.4p2 says that a disabled macro may never again be
833  // expanded, even if it's in a context where it could be expanded in the
834  // future.
836  if (MI->isObjectLike() || isNextPPTokenLParen())
837  Diag(Identifier, diag::pp_disabled_macro_expansion);
838  }
839  }
840  }
841 
842  // If this identifier is a keyword in a newer Standard or proposed Standard,
843  // produce a warning. Don't warn if we're not considering macro expansion,
844  // since this identifier might be the name of a macro.
845  // FIXME: This warning is disabled in cases where it shouldn't be, like
846  // "#define constexpr constexpr", "int constexpr;"
847  if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
848  Diag(Identifier, getIdentifierTable().getFutureCompatDiagKind(II, getLangOpts()))
849  << II.getName();
850  // Don't diagnose this keyword again in this translation unit.
851  II.setIsFutureCompatKeyword(false);
852  }
853 
854  // If this is an extension token, diagnose its use.
855  // We avoid diagnosing tokens that originate from macro definitions.
856  // FIXME: This warning is disabled in cases where it shouldn't be,
857  // like "#define TY typeof", "TY(1) x".
858  if (II.isExtensionToken() && !DisableMacroExpansion)
859  Diag(Identifier, diag::ext_token_used);
860 
861  // If this is the 'import' contextual keyword following an '@', note
862  // that the next token indicates a module name.
863  //
864  // Note that we do not treat 'import' as a contextual
865  // keyword when we're in a caching lexer, because caching lexers only get
866  // used in contexts where import declarations are disallowed.
867  //
868  // Likewise if this is the C++ Modules TS import keyword.
869  if (((LastTokenWasAt && II.isModulesImport()) ||
870  Identifier.is(tok::kw_import)) &&
871  !InMacroArgs && !DisableMacroExpansion &&
872  (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
873  CurLexerKind != CLK_CachingLexer) {
874  ModuleImportLoc = Identifier.getLocation();
875  NamedModuleImportPath.clear();
876  ModuleImportExpectsIdentifier = true;
877  CurLexerKind = CLK_LexAfterModuleImport;
878  }
879  return true;
880 }
881 
882 void Preprocessor::Lex(Token &Result) {
883  ++LexLevel;
884 
885  // We loop here until a lex function returns a token; this avoids recursion.
886  bool ReturnedToken;
887  do {
888  switch (CurLexerKind) {
889  case CLK_Lexer:
890  ReturnedToken = CurLexer->Lex(Result);
891  break;
892  case CLK_TokenLexer:
893  ReturnedToken = CurTokenLexer->Lex(Result);
894  break;
895  case CLK_CachingLexer:
896  CachingLex(Result);
897  ReturnedToken = true;
898  break;
899  case CLK_DependencyDirectivesLexer:
900  ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result);
901  break;
902  case CLK_LexAfterModuleImport:
903  ReturnedToken = LexAfterModuleImport(Result);
904  break;
905  }
906  } while (!ReturnedToken);
907 
908  if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure)
909  return;
910 
911  if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
912  // Remember the identifier before code completion token.
913  setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
915  // Set IdenfitierInfo to null to avoid confusing code that handles both
916  // identifiers and completion tokens.
917  Result.setIdentifierInfo(nullptr);
918  }
919 
920  // Update StdCXXImportSeqState to track our position within a C++20 import-seq
921  // if this token is being produced as a result of phase 4 of translation.
922  // Update TrackGMFState to decide if we are currently in a Global Module
923  // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state
924  // depends on the prevailing StdCXXImportSeq state in two cases.
925  if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
926  !Result.getFlag(Token::IsReinjected)) {
927  switch (Result.getKind()) {
928  case tok::l_paren: case tok::l_square: case tok::l_brace:
929  StdCXXImportSeqState.handleOpenBracket();
930  break;
931  case tok::r_paren: case tok::r_square:
932  StdCXXImportSeqState.handleCloseBracket();
933  break;
934  case tok::r_brace:
935  StdCXXImportSeqState.handleCloseBrace();
936  break;
937  // This token is injected to represent the translation of '#include "a.h"'
938  // into "import a.h;". Mimic the notional ';'.
939  case tok::annot_module_include:
940  case tok::semi:
941  TrackGMFState.handleSemi();
942  StdCXXImportSeqState.handleSemi();
943  break;
944  case tok::header_name:
945  case tok::annot_header_unit:
946  StdCXXImportSeqState.handleHeaderName();
947  break;
948  case tok::kw_export:
949  TrackGMFState.handleExport();
950  StdCXXImportSeqState.handleExport();
951  break;
952  case tok::identifier:
953  if (Result.getIdentifierInfo()->isModulesImport()) {
954  TrackGMFState.handleImport(StdCXXImportSeqState.afterTopLevelSeq());
955  StdCXXImportSeqState.handleImport();
956  if (StdCXXImportSeqState.afterImportSeq()) {
957  ModuleImportLoc = Result.getLocation();
958  NamedModuleImportPath.clear();
959  ModuleImportExpectsIdentifier = true;
960  CurLexerKind = CLK_LexAfterModuleImport;
961  }
962  break;
963  } else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) {
964  TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
965  break;
966  }
967  [[fallthrough]];
968  default:
969  TrackGMFState.handleMisc();
970  StdCXXImportSeqState.handleMisc();
971  break;
972  }
973  }
974 
975  LastTokenWasAt = Result.is(tok::at);
976  --LexLevel;
977 
978  if ((LexLevel == 0 || PreprocessToken) &&
979  !Result.getFlag(Token::IsReinjected)) {
980  if (LexLevel == 0)
981  ++TokenCount;
982  if (OnToken)
983  OnToken(Result);
984  }
985 }
986 
987 /// Lex a header-name token (including one formed from header-name-tokens if
988 /// \p AllowConcatenation is \c true).
989 ///
990 /// \param FilenameTok Filled in with the next token. On success, this will
991 /// be either a header_name token. On failure, it will be whatever other
992 /// token was found instead.
993 /// \param AllowMacroExpansion If \c true, allow the header name to be formed
994 /// by macro expansion (concatenating tokens as necessary if the first
995 /// token is a '<').
996 /// \return \c true if we reached EOD or EOF while looking for a > token in
997 /// a concatenated header name and diagnosed it. \c false otherwise.
998 bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
999  // Lex using header-name tokenization rules if tokens are being lexed from
1000  // a file. Just grab a token normally if we're in a macro expansion.
1001  if (CurPPLexer)
1002  CurPPLexer->LexIncludeFilename(FilenameTok);
1003  else
1004  Lex(FilenameTok);
1005 
1006  // This could be a <foo/bar.h> file coming from a macro expansion. In this
1007  // case, glue the tokens together into an angle_string_literal token.
1008  SmallString<128> FilenameBuffer;
1009  if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
1010  bool StartOfLine = FilenameTok.isAtStartOfLine();
1011  bool LeadingSpace = FilenameTok.hasLeadingSpace();
1012  bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1013 
1014  SourceLocation Start = FilenameTok.getLocation();
1016  FilenameBuffer.push_back('<');
1017 
1018  // Consume tokens until we find a '>'.
1019  // FIXME: A header-name could be formed starting or ending with an
1020  // alternative token. It's not clear whether that's ill-formed in all
1021  // cases.
1022  while (FilenameTok.isNot(tok::greater)) {
1023  Lex(FilenameTok);
1024  if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
1025  Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
1026  Diag(Start, diag::note_matching) << tok::less;
1027  return true;
1028  }
1029 
1030  End = FilenameTok.getLocation();
1031 
1032  // FIXME: Provide code completion for #includes.
1033  if (FilenameTok.is(tok::code_completion)) {
1035  Lex(FilenameTok);
1036  continue;
1037  }
1038 
1039  // Append the spelling of this token to the buffer. If there was a space
1040  // before it, add it now.
1041  if (FilenameTok.hasLeadingSpace())
1042  FilenameBuffer.push_back(' ');
1043 
1044  // Get the spelling of the token, directly into FilenameBuffer if
1045  // possible.
1046  size_t PreAppendSize = FilenameBuffer.size();
1047  FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
1048 
1049  const char *BufPtr = &FilenameBuffer[PreAppendSize];
1050  unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
1051 
1052  // If the token was spelled somewhere else, copy it into FilenameBuffer.
1053  if (BufPtr != &FilenameBuffer[PreAppendSize])
1054  memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1055 
1056  // Resize FilenameBuffer to the correct size.
1057  if (FilenameTok.getLength() != ActualLen)
1058  FilenameBuffer.resize(PreAppendSize + ActualLen);
1059  }
1060 
1061  FilenameTok.startToken();
1062  FilenameTok.setKind(tok::header_name);
1063  FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine);
1064  FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace);
1065  FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro);
1066  CreateString(FilenameBuffer, FilenameTok, Start, End);
1067  } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
1068  // Convert a string-literal token of the form " h-char-sequence "
1069  // (produced by macro expansion) into a header-name token.
1070  //
1071  // The rules for header-names don't quite match the rules for
1072  // string-literals, but all the places where they differ result in
1073  // undefined behavior, so we can and do treat them the same.
1074  //
1075  // A string-literal with a prefix or suffix is not translated into a
1076  // header-name. This could theoretically be observable via the C++20
1077  // context-sensitive header-name formation rules.
1078  StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
1079  if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1080  FilenameTok.setKind(tok::header_name);
1081  }
1082 
1083  return false;
1084 }
1085 
1086 /// Collect the tokens of a C++20 pp-import-suffix.
1088  // FIXME: For error recovery, consider recognizing attribute syntax here
1089  // and terminating / diagnosing a missing semicolon if we find anything
1090  // else? (Can we leave that to the parser?)
1091  unsigned BracketDepth = 0;
1092  while (true) {
1093  Toks.emplace_back();
1094  Lex(Toks.back());
1095 
1096  switch (Toks.back().getKind()) {
1097  case tok::l_paren: case tok::l_square: case tok::l_brace:
1098  ++BracketDepth;
1099  break;
1100 
1101  case tok::r_paren: case tok::r_square: case tok::r_brace:
1102  if (BracketDepth == 0)
1103  return;
1104  --BracketDepth;
1105  break;
1106 
1107  case tok::semi:
1108  if (BracketDepth == 0)
1109  return;
1110  break;
1111 
1112  case tok::eof:
1113  return;
1114 
1115  default:
1116  break;
1117  }
1118  }
1119 }
1120 
1121 
1122 /// Lex a token following the 'import' contextual keyword.
1123 ///
1124 /// pp-import: [C++20]
1125 /// import header-name pp-import-suffix[opt] ;
1126 /// import header-name-tokens pp-import-suffix[opt] ;
1127 /// [ObjC] @ import module-name ;
1128 /// [Clang] import module-name ;
1129 ///
1130 /// header-name-tokens:
1131 /// string-literal
1132 /// < [any sequence of preprocessing-tokens other than >] >
1133 ///
1134 /// module-name:
1135 /// module-name-qualifier[opt] identifier
1136 ///
1137 /// module-name-qualifier
1138 /// module-name-qualifier[opt] identifier .
1139 ///
1140 /// We respond to a pp-import by importing macros from the named module.
1142  // Figure out what kind of lexer we actually have.
1144 
1145  // Lex the next token. The header-name lexing rules are used at the start of
1146  // a pp-import.
1147  //
1148  // For now, we only support header-name imports in C++20 mode.
1149  // FIXME: Should we allow this in all language modes that support an import
1150  // declaration as an extension?
1151  if (NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1152  if (LexHeaderName(Result))
1153  return true;
1154  } else {
1155  Lex(Result);
1156  }
1157 
1158  // Allocate a holding buffer for a sequence of tokens and introduce it into
1159  // the token stream.
1160  auto EnterTokens = [this](ArrayRef<Token> Toks) {
1161  auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1162  std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1163  EnterTokenStream(std::move(ToksCopy), Toks.size(),
1164  /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1165  };
1166 
1167  // Check for a header-name.
1168  SmallVector<Token, 32> Suffix;
1169  if (Result.is(tok::header_name)) {
1170  // Enter the header-name token into the token stream; a Lex action cannot
1171  // both return a token and cache tokens (doing so would corrupt the token
1172  // cache if the call to Lex comes from CachingLex / PeekAhead).
1173  Suffix.push_back(Result);
1174 
1175  // Consume the pp-import-suffix and expand any macros in it now. We'll add
1176  // it back into the token stream later.
1177  CollectPpImportSuffix(Suffix);
1178  if (Suffix.back().isNot(tok::semi)) {
1179  // This is not a pp-import after all.
1180  EnterTokens(Suffix);
1181  return false;
1182  }
1183 
1184  // C++2a [cpp.module]p1:
1185  // The ';' preprocessing-token terminating a pp-import shall not have
1186  // been produced by macro replacement.
1187  SourceLocation SemiLoc = Suffix.back().getLocation();
1188  if (SemiLoc.isMacroID())
1189  Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1190 
1191  // Reconstitute the import token.
1192  Token ImportTok;
1193  ImportTok.startToken();
1194  ImportTok.setKind(tok::kw_import);
1195  ImportTok.setLocation(ModuleImportLoc);
1196  ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
1197  ImportTok.setLength(6);
1198 
1199  auto Action = HandleHeaderIncludeOrImport(
1200  /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
1201  switch (Action.Kind) {
1202  case ImportAction::None:
1203  break;
1204 
1205  case ImportAction::ModuleBegin:
1206  // Let the parser know we're textually entering the module.
1207  Suffix.emplace_back();
1208  Suffix.back().startToken();
1209  Suffix.back().setKind(tok::annot_module_begin);
1210  Suffix.back().setLocation(SemiLoc);
1211  Suffix.back().setAnnotationEndLoc(SemiLoc);
1212  Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1213  [[fallthrough]];
1214 
1215  case ImportAction::ModuleImport:
1216  case ImportAction::HeaderUnitImport:
1217  case ImportAction::SkippedModuleImport:
1218  // We chose to import (or textually enter) the file. Convert the
1219  // header-name token into a header unit annotation token.
1220  Suffix[0].setKind(tok::annot_header_unit);
1221  Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1222  Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1223  // FIXME: Call the moduleImport callback?
1224  break;
1225  case ImportAction::Failure:
1226  assert(TheModuleLoader.HadFatalFailure &&
1227  "This should be an early exit only to a fatal error");
1228  Result.setKind(tok::eof);
1229  CurLexer->cutOffLexing();
1230  EnterTokens(Suffix);
1231  return true;
1232  }
1233 
1234  EnterTokens(Suffix);
1235  return false;
1236  }
1237 
1238  // The token sequence
1239  //
1240  // import identifier (. identifier)*
1241  //
1242  // indicates a module import directive. We already saw the 'import'
1243  // contextual keyword, so now we're looking for the identifiers.
1244  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1245  // We expected to see an identifier here, and we did; continue handling
1246  // identifiers.
1247  NamedModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
1248  Result.getLocation()));
1249  ModuleImportExpectsIdentifier = false;
1250  CurLexerKind = CLK_LexAfterModuleImport;
1251  return true;
1252  }
1253 
1254  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1255  // see the next identifier. (We can also see a '[[' that begins an
1256  // attribute-specifier-seq here under the C++ Modules TS.)
1257  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1258  ModuleImportExpectsIdentifier = true;
1259  CurLexerKind = CLK_LexAfterModuleImport;
1260  return true;
1261  }
1262 
1263  // If we didn't recognize a module name at all, this is not a (valid) import.
1264  if (NamedModuleImportPath.empty() || Result.is(tok::eof))
1265  return true;
1266 
1267  // Consume the pp-import-suffix and expand any macros in it now, if we're not
1268  // at the semicolon already.
1269  SourceLocation SemiLoc = Result.getLocation();
1270  if (Result.isNot(tok::semi)) {
1271  Suffix.push_back(Result);
1272  CollectPpImportSuffix(Suffix);
1273  if (Suffix.back().isNot(tok::semi)) {
1274  // This is not an import after all.
1275  EnterTokens(Suffix);
1276  return false;
1277  }
1278  SemiLoc = Suffix.back().getLocation();
1279  }
1280 
1281  // Under the Modules TS, the dot is just part of the module name, and not
1282  // a real hierarchy separator. Flatten such module names now.
1283  //
1284  // FIXME: Is this the right level to be performing this transformation?
1285  std::string FlatModuleName;
1286  if (getLangOpts().ModulesTS || getLangOpts().CPlusPlusModules) {
1287  for (auto &Piece : NamedModuleImportPath) {
1288  if (!FlatModuleName.empty())
1289  FlatModuleName += ".";
1290  FlatModuleName += Piece.first->getName();
1291  }
1292  SourceLocation FirstPathLoc = NamedModuleImportPath[0].second;
1293  NamedModuleImportPath.clear();
1294  NamedModuleImportPath.push_back(
1295  std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1296  }
1297 
1298  Module *Imported = nullptr;
1299  if (getLangOpts().Modules) {
1300  Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1301  NamedModuleImportPath,
1303  /*IsInclusionDirective=*/false);
1304  if (Imported)
1305  makeModuleVisible(Imported, SemiLoc);
1306  }
1307  if (Callbacks)
1308  Callbacks->moduleImport(ModuleImportLoc, NamedModuleImportPath, Imported);
1309 
1310  if (!Suffix.empty()) {
1311  EnterTokens(Suffix);
1312  return false;
1313  }
1314  return true;
1315 }
1316 
1318  CurSubmoduleState->VisibleModules.setVisible(
1319  M, Loc, [](Module *) {},
1320  [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1321  // FIXME: Include the path in the diagnostic.
1322  // FIXME: Include the import location for the conflicting module.
1323  Diag(ModuleImportLoc, diag::warn_module_conflict)
1324  << Path[0]->getFullModuleName()
1325  << Conflict->getFullModuleName()
1326  << Message;
1327  });
1328 
1329  // Add this module to the imports list of the currently-built submodule.
1330  if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1331  BuildingSubmoduleStack.back().M->Imports.insert(M);
1332 }
1333 
1335  const char *DiagnosticTag,
1336  bool AllowMacroExpansion) {
1337  // We need at least one string literal.
1338  if (Result.isNot(tok::string_literal)) {
1339  Diag(Result, diag::err_expected_string_literal)
1340  << /*Source='in...'*/0 << DiagnosticTag;
1341  return false;
1342  }
1343 
1344  // Lex string literal tokens, optionally with macro expansion.
1345  SmallVector<Token, 4> StrToks;
1346  do {
1347  StrToks.push_back(Result);
1348 
1349  if (Result.hasUDSuffix())
1350  Diag(Result, diag::err_invalid_string_udl);
1351 
1352  if (AllowMacroExpansion)
1353  Lex(Result);
1354  else
1355  LexUnexpandedToken(Result);
1356  } while (Result.is(tok::string_literal));
1357 
1358  // Concatenate and parse the strings.
1359  StringLiteralParser Literal(StrToks, *this);
1360  assert(Literal.isOrdinary() && "Didn't allow wide strings in");
1361 
1362  if (Literal.hadError)
1363  return false;
1364 
1365  if (Literal.Pascal) {
1366  Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1367  << /*Source='in...'*/0 << DiagnosticTag;
1368  return false;
1369  }
1370 
1371  String = std::string(Literal.GetString());
1372  return true;
1373 }
1374 
1376  assert(Tok.is(tok::numeric_constant));
1377  SmallString<8> IntegerBuffer;
1378  bool NumberInvalid = false;
1379  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1380  if (NumberInvalid)
1381  return false;
1384  getDiagnostics());
1385  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1386  return false;
1387  llvm::APInt APVal(64, 0);
1388  if (Literal.GetIntegerValue(APVal))
1389  return false;
1390  Lex(Tok);
1391  Value = APVal.getLimitedValue();
1392  return true;
1393 }
1394 
1396  assert(Handler && "NULL comment handler");
1397  assert(!llvm::is_contained(CommentHandlers, Handler) &&
1398  "Comment handler already registered");
1399  CommentHandlers.push_back(Handler);
1400 }
1401 
1403  std::vector<CommentHandler *>::iterator Pos =
1404  llvm::find(CommentHandlers, Handler);
1405  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1406  CommentHandlers.erase(Pos);
1407 }
1408 
1410  bool AnyPendingTokens = false;
1411  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1412  HEnd = CommentHandlers.end();
1413  H != HEnd; ++H) {
1414  if ((*H)->HandleComment(*this, Comment))
1415  AnyPendingTokens = true;
1416  }
1417  if (!AnyPendingTokens || getCommentRetentionState())
1418  return false;
1419  Lex(result);
1420  return true;
1421 }
1422 
1423 void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1424  const MacroAnnotations &A =
1425  getMacroAnnotations(Identifier.getIdentifierInfo());
1426  assert(A.DeprecationInfo &&
1427  "Macro deprecation warning without recorded annotation!");
1428  const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1429  if (Info.Message.empty())
1430  Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1431  << Identifier.getIdentifierInfo() << 0;
1432  else
1433  Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1434  << Identifier.getIdentifierInfo() << 1 << Info.Message;
1435  Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1436 }
1437 
1438 void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1439  const MacroAnnotations &A =
1440  getMacroAnnotations(Identifier.getIdentifierInfo());
1441  assert(A.RestrictExpansionInfo &&
1442  "Macro restricted expansion warning without recorded annotation!");
1443  const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1444  if (Info.Message.empty())
1445  Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1446  << Identifier.getIdentifierInfo() << 0;
1447  else
1448  Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1449  << Identifier.getIdentifierInfo() << 1 << Info.Message;
1450  Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1451 }
1452 
1453 void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1454  bool IsUndef) const {
1455  const MacroAnnotations &A =
1456  getMacroAnnotations(Identifier.getIdentifierInfo());
1457  assert(A.FinalAnnotationLoc &&
1458  "Final macro warning without recorded annotation!");
1459 
1460  Diag(Identifier, diag::warn_pragma_final_macro)
1461  << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1462  Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1463 }
1464 
1465 ModuleLoader::~ModuleLoader() = default;
1466 
1468 
1470 
1472 
1474  if (Record)
1475  return;
1476 
1477  Record = new PreprocessingRecord(getSourceManager());
1478  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1479 }
clang::Token::startToken
void startToken()
Reset all flags to cleared.
Definition: Token.h:171
clang::MacroDirective::DefInfo::getLocation
SourceLocation getLocation() const
Definition: MacroInfo.h:485
clang::MacroInfo::getNumTokens
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:235
PreprocessorLexer.h
Builtins.h
clang::MacroInfo::getReplacementToken
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:237
clang::Preprocessor::CreateString
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
Definition: Preprocessor.cpp:489
clang::Token::IsReinjected
@ IsReinjected
Definition: Token.h:88
clang::Preprocessor::PrintStats
void PrintStats()
Definition: Preprocessor.cpp:274
clang::Preprocessor::FinalizeForModelFile
void FinalizeForModelFile()
Cleanup after model file parsing.
Definition: Preprocessor.cpp:229
clang::Token::hasUCN
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:298
clang::Token::hasLeadingSpace
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:274
clang::interp::APInt
llvm::APInt APInt
Definition: Integral.h:29
clang::Preprocessor::FinishLexStringLiteral
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
Definition: Preprocessor.cpp:1334
clang::Preprocessor::setCodeCompletionReached
void setCodeCompletionReached()
Note that we hit the code-completion point.
Definition: Preprocessor.h:1825
clang::Preprocessor::getMacroDefinition
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:1212
clang::Decl::getEndLoc
SourceLocation getEndLoc() const LLVM_READONLY
Definition: DeclBase.h:428
clang::DeclaratorContext::File
@ File
clang::Preprocessor::SetCodeCompletionPoint
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
Definition: Preprocessor.cpp:395
clang::SourceRange
A trivial tuple used to represent a source range.
Definition: SourceLocation.h:210
string
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD) string(SUBSTRING $
Definition: CMakeLists.txt:23
clang::Preprocessor::getLastMacroWithSpelling
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
Definition: Preprocessor.cpp:357
clang::Preprocessor::setCodeCompletionTokenRange
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Definition: Preprocessor.h:1447
clang::Preprocessor::Lex
void Lex(Token &Result)
Lex the next token for this preprocessor.
Definition: Preprocessor.cpp:882
clang::Preprocessor::HandleIdentifier
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Definition: Preprocessor.cpp:790
clang::MacroInfo::isObjectLike
bool isObjectLike() const
Definition: MacroInfo.h:202
clang::Preprocessor::macro_iterator
MacroMap::const_iterator macro_iterator
Definition: Preprocessor.h:1313
clang::MacroDefinition
A description of the current definition of a macro.
Definition: MacroInfo.h:587
clang::Preprocessor::LexHeaderName
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
Definition: Preprocessor.cpp:998
llvm::SmallVector
Definition: LLVM.h:38
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::Preprocessor::TUKind
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:287
clang::MacroInfo::tokens_begin
const_tokens_iterator tokens_begin() const
Definition: MacroInfo.h:244
clang::SourceLocation::getLocWithOffset
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
Definition: SourceLocation.h:134
clang::Preprocessor::getCommentRetentionState
bool getCommentRetentionState() const
Definition: Preprocessor.h:1105
TargetInfo.h
clang::MultiVersionKind::Target
@ Target
clang::Token::isExpandDisabled
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6....
Definition: Token.h:278
clang::PreprocessorLexer::setConditionalLevels
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
Definition: PreprocessorLexer.h:175
clang::Token::getIdentifierInfo
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:181
clang::MacroArgs
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
clang::DiagnosticsEngine
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
clang::TargetInfo
Exposes information about the current target.
Definition: TargetInfo.h:205
memcpy
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
Definition: __clang_cuda_device_functions.h:1549
clang::PreprocessingRecord
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
Definition: PreprocessingRecord.h:305
Module.h
clang::Token::isAtStartOfLine
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:270
clang::IdentifierInfo::isOutOfDate
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Definition: IdentifierTable.h:415
clang::Preprocessor::CodeCompleteIncludedFile
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
Definition: Preprocessor.cpp:451
clang::Preprocessor::macro_end
macro_iterator macro_end(bool IncludeExternalMacros=true) const
Definition: Preprocessor.cpp:340
clang::ModuleLoader::HadFatalFailure
bool HadFatalFailure
Definition: ModuleLoader.h:162
clang::MacroInfo
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
SourceManager.h
clang::Token::setIdentifierInfo
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:190
clang::MacroDirective::DefInfo::getMacroInfo
MacroInfo * getMacroInfo()
Definition: MacroInfo.h:491
Identifier
StringRef Identifier
Definition: Format.cpp:2780
clang::IdentifierInfo::getTokenID
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
Definition: IdentifierTable.h:262
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
LiteralSupport.h
clang::CommentHandler
Abstract base class that describes a handler that will receive source ranges for each of the comments...
Definition: Preprocessor.h:2695
clang::CustomizableOptional< FileEntryRef >
End
SourceLocation End
Definition: USRLocFinder.cpp:167
TokenLexer.h
clang::Preprocessor::getIdentifierTable
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:1067
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:638
clang::Preprocessor::getLangOpts
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:1060
Preprocessor.h
clang::HeaderSearch
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:223
clang::ModuleLoader::loadModule
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
clang::Preprocessor::creatingPCHWithPragmaHdrStop
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Definition: Preprocessor.cpp:627
clang::TU_Prefix
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:923
clang::Preprocessor::usingPCHWithThroughHeader
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
Definition: Preprocessor.cpp:622
clang::PreprocessorLexer::LexIncludeFilename
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
Definition: PreprocessorLexer.cpp:32
clang::dataflow::Literal
uint32_t Literal
Literals are represented as positive integers.
Definition: WatchedLiteralsSolver.cpp:55
clang::Preprocessor::getCurrentModuleImplementation
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
Definition: Preprocessor.cpp:532
clang::CodeCompletionHandler::CodeCompleteIncludedFile
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an include directive.
Definition: CodeCompletionHandler.h:67
clang::SourceLocation::print
void print(raw_ostream &OS, const SourceManager &SM) const
Definition: SourceLocation.cpp:62
clang::Preprocessor::SetPoisonReason
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
Definition: Preprocessor.cpp:748
clang::Token::setRawIdentifierData
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:211
clang::Module
Describes a module or submodule.
Definition: Module.h:98
clang::Module::Hidden
@ Hidden
All of the names in this module are hidden.
Definition: Module.h:337
clang::Preprocessor::EnterSourceFile
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Definition: PPLexerChange.cpp:69
clang::Preprocessor::EndSourceFile
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
Definition: Preprocessor.cpp:700
clang::Decl::getKind
Kind getKind() const
Definition: DeclBase.h:435
clang::Preprocessor::LookupFile
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
Definition: PPDirectives.cpp:932
hlsl::uint64_t
unsigned long uint64_t
Definition: hlsl_basic_types.h:25
clang::XRayInstrKind::None
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
Pragma.h
clang::Preprocessor::InitializeForModelFile
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
Definition: Preprocessor.cpp:217
clang::Preprocessor::addCommentHandler
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
Definition: Preprocessor.cpp:1395
clang::Token::setFlagValue
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:261
clang::Token::getKind
tok::TokenKind getKind() const
Definition: Token.h:93
clang::IdentifierInfo::setIsFutureCompatKeyword
void setIsFutureCompatKeyword(bool Val)
Definition: IdentifierTable.h:336
clang::ExternalPreprocessorSource::~ExternalPreprocessorSource
virtual ~ExternalPreprocessorSource()
clang::SourceManager::getMainFileID
FileID getMainFileID() const
Returns the FileID of the main source file.
Definition: SourceManager.h:852
clang::FileID::isValid
bool isValid() const
Definition: SourceLocation.h:44
clang::IdentifierInfo::setIsPoisoned
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
Definition: IdentifierTable.h:346
LangOptions.h
clang::Preprocessor::isPCHThroughHeader
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
Definition: Preprocessor.cpp:611
clang::Preprocessor::setTUFPEvalMethod
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2216
LexDiagnostic.h
clang::Preprocessor::Diag
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
Definition: Preprocessor.h:1897
ModuleLoader.h
clang::Token::is
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:98
clang::HeaderSearch::lookupModule
Module * lookupModule(StringRef ModuleName, SourceLocation ImportLoc=SourceLocation(), bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
Definition: HeaderSearch.cpp:272
clang::Preprocessor::getTargetInfo
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:1061
clang::Preprocessor::createPreprocessingRecord
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
Definition: Preprocessor.cpp:1473
clang::Token::DisableExpand
@ DisableExpand
Definition: Token.h:78
IdentifierTable.h
clang::tok::getTokenName
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:24
llvm::SmallString
Definition: LLVM.h:37
clang::Token::needsCleaning
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:287
clang::Token::LeadingEmptyMacro
@ LeadingEmptyMacro
Definition: Token.h:80
clang::SourceManager::getMemoryBufferForFileOrNone
std::optional< llvm::MemoryBufferRef > getMemoryBufferForFileOrNone(const FileEntry *File)
Retrieve the memory buffer associated with the given file.
Definition: SourceManager.cpp:685
clang::Preprocessor::LexAfterModuleImport
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
Definition: Preprocessor.cpp:1141
clang::CommentHandler::~CommentHandler
virtual ~CommentHandler()
clang::ExternalPreprocessorSource::ReadDefinedMacros
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
clang::SrcMgr::C_User
@ C_User
Definition: SourceManager.h:81
clang::Preprocessor::EnterMainSourceFile
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
Definition: Preprocessor.cpp:545
Line
const AnnotatedLine * Line
Definition: UsingDeclarationsSorter.cpp:95
clang::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: Token.h:99
clang::FileEntry
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:356
clang::Preprocessor::SplitToken
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Definition: Preprocessor.cpp:509
clang::Token::isLiteral
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
clang::Token::getLength
unsigned getLength() const
Definition: Token.h:129
clang::SourceManager::isBeforeInTranslationUnit
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
Definition: SourceManager.cpp:2025
clang::Token::LeadingSpace
@ LeadingSpace
Definition: Token.h:76
clang::Preprocessor::getSourceManager
SourceManager & getSourceManager() const
Definition: Preprocessor.h:1064
clang::LangOptions::FEM_UnsetOnCommandLine
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Definition: LangOptions.h:296
clang::Preprocessor::HandleComment
bool HandleComment(Token &result, SourceRange Comment)
Definition: Preprocessor.cpp:1409
clang::LangOptions::FEM_Indeterminable
@ FEM_Indeterminable
The evaluation method cannot be determined or is inconsistent for this target.
Definition: LangOptions.h:285
clang::Preprocessor::getSpelling
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
Definition: Preprocessor.h:1912
MacroArgs.h
clang::Preprocessor::PoisonSEHIdentifiers
void PoisonSEHIdentifiers(bool Poison=true)
Definition: Preprocessor.cpp:752
clang::expandUCNs
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
Definition: LiteralSupport.cpp:331
clang::Preprocessor::CollectPpImportSuffix
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
Definition: Preprocessor.cpp:1087
FileSystemStatCache.h
SourceLocation.h
clang::Preprocessor::SkipTokensWhileUsingPCH
void SkipTokensWhileUsingPCH()
Skip tokens until after the include of the through header or until after a #pragma hdrstop.
Definition: Preprocessor.cpp:640
clang::Preprocessor::getTotalMemory
size_t getTotalMemory() const
Definition: Preprocessor.cpp:327
clang::Preprocessor::CodeCompleteNaturalLanguage
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Definition: Preprocessor.cpp:458
clang::Token::setLiteralData
void setLiteralData(const char *Ptr)
Definition: Token.h:223
clang::ScratchBuffer
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Definition: ScratchBuffer.h:24
clang::Preprocessor::DumpLocation
void DumpLocation(SourceLocation Loc) const
Definition: Preprocessor.cpp:261
MacroDefinitionEquals
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
Definition: Preprocessor.cpp:351
clang::Preprocessor::usingPCHWithPragmaHdrStop
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
Definition: Preprocessor.cpp:631
clang::IdentifierInfo::isPoisoned
bool isPoisoned() const
Return true if this token has been poisoned.
Definition: IdentifierTable.h:355
PreprocessingRecord.h
llvm::ArrayRef
Definition: LLVM.h:34
Lexer.h
Value
Value
Definition: UninitializedValues.cpp:102
clang::Preprocessor::setCodeCompletionIdentifierInfo
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
Definition: Preprocessor.h:1441
clang::SourceManager::getCharacterData
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer.
Definition: SourceManager.cpp:1159
HeaderSearch.h
LLVM.h
clang::Preprocessor::DumpToken
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
Definition: Preprocessor.cpp:235
clang::Token::StartOfLine
@ StartOfLine
Definition: Token.h:74
clang::Preprocessor::setCurrentFPEvalMethod
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
Definition: Preprocessor.h:2205
clang::NumericLiteralParser
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber,...
Definition: LiteralSupport.h:42
clang::Preprocessor::getCurrentModule
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
Definition: Preprocessor.cpp:525
clang::ModuleLoader
Abstract interface for a module loader.
Definition: ModuleLoader.h:82
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:85
clang::SourceManager::overrideFileContents
void overrideFileContents(const FileEntry *SourceFile, const llvm::MemoryBufferRef &Buffer)
Override the contents of the given source file by providing an already-allocated buffer.
Definition: SourceManager.h:963
clang::IdentifierTable::AddKeywords
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
Definition: IdentifierTable.cpp:284
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:126
clang::SourceLocation::isMacroID
bool isMacroID() const
Definition: SourceLocation.h:103
clang::LangOptions
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:82
CodeCompletionHandler.h
clang::Token::isAnnotation
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
clang::Preprocessor::removeCommentHandler
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Definition: Preprocessor.cpp:1402
clang::SourceManager::isLoadedFileID
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
Definition: SourceManager.h:1760
clang::Token::setLocation
void setLocation(SourceLocation L)
Definition: Token.h:134
std
Definition: Format.h:4605
clang::TranslationUnitKind
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:917
clang::IdentifierInfo::getName
StringRef getName() const
Return the actual identifier string.
Definition: IdentifierTable.h:196
clang::SourceLocation::isInvalid
bool isInvalid() const
Definition: SourceLocation.h:111
clang
Definition: CalledOnceCheck.h:17
clang::Preprocessor::addPPCallbacks
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:1158
clang::SourceManager::createFileID
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
Definition: SourceManager.cpp:533
clang::ExternalPreprocessorSource::updateOutOfDateIdentifier
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
clang::Preprocessor::markIncluded
bool markIncluded(const FileEntry *File)
Mark the file as included.
Definition: Preprocessor.h:1346
clang::CodeCompletionHandler::CodeCompleteNaturalLanguage
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Definition: CodeCompletionHandler.h:72
clang::Preprocessor::creatingPCHWithThroughHeader
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
Definition: Preprocessor.cpp:617
clang::Preprocessor::LookUpIdentifierInfo
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Definition: Preprocessor.cpp:713
clang::PragmaHandlerRegistry
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Definition: Preprocessor.h:2715
MacroInfo.h
clang::Preprocessor::getDiagnostics
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:1057
clang::Preprocessor::Preprocessor
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
Definition: Preprocessor.cpp:79
clang::EmptylineHandler::~EmptylineHandler
virtual ~EmptylineHandler()
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:38
clang::SourceLocation::isValid
bool isValid() const
Return true if this is a valid SourceLocation object.
Definition: SourceLocation.h:110
clang::Token::setLength
void setLength(unsigned Len)
Definition: Token.h:135
clang::FileID::isInvalid
bool isInvalid() const
Definition: SourceLocation.h:45
clang::SourceManager::getFileEntryForID
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
Definition: SourceManager.h:1046
clang::Token::setKind
void setKind(tok::TokenKind K)
Definition: Token.h:94
clang::IdentifierInfoLookup
Provides lookups to, and iteration over, IdentiferInfo objects.
Definition: IdentifierTable.h:535
clang::Preprocessor::makeModuleVisible
void makeModuleVisible(Module *M, SourceLocation Loc)
Definition: Preprocessor.cpp:1317
clang::Preprocessor::recomputeCurLexerKind
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Definition: Preprocessor.cpp:384
clang::Preprocessor::DumpMacro
void DumpMacro(const MacroInfo &MI) const
Definition: Preprocessor.cpp:265
clang::IdentifierInfo::isModulesImport
bool isModulesImport() const
Determine whether this is the contextual keyword import.
Definition: IdentifierTable.h:428
clang::IdentifierInfo::isExtensionToken
bool isExtensionToken() const
get/setExtension - Initialize information about whether or not this language token is an extension.
Definition: IdentifierTable.h:321
clang::Token::isOneOf
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:100
clang::IdentifierInfo::isFutureCompatKeyword
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Definition: IdentifierTable.h:335
ScratchBuffer.h
clang::CodeCompletionHandler::~CodeCompletionHandler
virtual ~CodeCompletionHandler()
clang::comments::tok::eof
@ eof
Definition: CommentLexer.h:33
llvm::SmallVectorImpl< char >
FileManager.h
clang::PragmaNamespace
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:96
clang::Preprocessor::LexUnexpandedToken
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Definition: Preprocessor.h:1627
true
#define true
Definition: stdbool.h:21
PreprocessorOptions.h
SM
#define SM(sm)
Definition: Cuda.cpp:78
clang::Preprocessor::HandlePoisonedIdentifier
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
Definition: Preprocessor.cpp:766
clang::Preprocessor::getHeaderSearchInfo
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:1065
Token.h
clang::MacroDirective::DefInfo
Definition: MacroInfo.h:361
clang::StringLiteralParser
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
Definition: LiteralSupport.h:218
clang::Preprocessor::getExternalSource
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:1077
clang::Decl::getLocation
SourceLocation getLocation() const
Definition: DeclBase.h:432
clang::Preprocessor::getMacroAnnotations
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
Definition: Preprocessor.h:2670
clang::Preprocessor::~Preprocessor
~Preprocessor()
Definition: Preprocessor.cpp:164
clang::Preprocessor::getPredefinesFileID
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:1150
clang::HeaderSearch::setTarget
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Definition: HeaderSearch.cpp:744
clang::SourceManager::createExpansionLoc
SourceLocation createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned Length, bool ExpansionIsTokenRange=true, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Creates an expansion SLocEntry for a macro use.
Definition: SourceManager.cpp:641
clang::ModuleMacro
Represents a macro directive exported by a module.
Definition: MacroInfo.h:511
clang::Preprocessor::getIdentifierInfo
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Definition: Preprocessor.h:1378
clang::Preprocessor::macro_begin
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Definition: Preprocessor.cpp:313
clang::Preprocessor::parseSimpleIntegerLiteral
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
Definition: Preprocessor.cpp:1375
ExternalPreprocessorSource.h
clang::Preprocessor::Initialize
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Definition: Preprocessor.cpp:184
clang::ModuleLoader::~ModuleLoader
virtual ~ModuleLoader()
clang::Token::hasLeadingEmptyMacro
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
Definition: Token.h:291