clang  10.0.0svn
Preprocessor.cpp
Go to the documentation of this file.
1 //===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Preprocessor interface.
10 //
11 //===----------------------------------------------------------------------===//
12 //
13 // Options to support:
14 // -H - Print the name of each header file used.
15 // -d[DNI] - Dump various things.
16 // -fworking-directory - #line's with preprocessor's working dir.
17 // -fpreprocessed
18 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19 // -W*
20 // -w
21 //
22 // Messages to emit:
23 // "Multiple include guards may be useful for:\n"
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "clang/Lex/Preprocessor.h"
31 #include "clang/Basic/LLVM.h"
33 #include "clang/Basic/Module.h"
36 #include "clang/Basic/TargetInfo.h"
39 #include "clang/Lex/HeaderSearch.h"
41 #include "clang/Lex/Lexer.h"
43 #include "clang/Lex/MacroArgs.h"
44 #include "clang/Lex/MacroInfo.h"
45 #include "clang/Lex/ModuleLoader.h"
46 #include "clang/Lex/Pragma.h"
51 #include "clang/Lex/Token.h"
52 #include "clang/Lex/TokenLexer.h"
53 #include "llvm/ADT/APInt.h"
54 #include "llvm/ADT/ArrayRef.h"
55 #include "llvm/ADT/DenseMap.h"
56 #include "llvm/ADT/SmallString.h"
57 #include "llvm/ADT/SmallVector.h"
58 #include "llvm/ADT/STLExtras.h"
59 #include "llvm/ADT/StringRef.h"
60 #include "llvm/ADT/StringSwitch.h"
61 #include "llvm/Support/Capacity.h"
62 #include "llvm/Support/ErrorHandling.h"
63 #include "llvm/Support/MemoryBuffer.h"
64 #include "llvm/Support/raw_ostream.h"
65 #include <algorithm>
66 #include <cassert>
67 #include <memory>
68 #include <string>
69 #include <utility>
70 #include <vector>
71 
72 using namespace clang;
73 
74 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
75 
77 
78 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
79  DiagnosticsEngine &diags, LangOptions &opts,
80  SourceManager &SM, HeaderSearch &Headers,
81  ModuleLoader &TheModuleLoader,
82  IdentifierInfoLookup *IILookup, bool OwnsHeaders,
83  TranslationUnitKind TUKind)
84  : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
85  FileMgr(Headers.getFileMgr()), SourceMgr(SM),
86  ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
87  TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
88  // As the language options may have not been loaded yet (when
89  // deserializing an ASTUnit), adding keywords to the identifier table is
90  // deferred to Preprocessor::Initialize().
91  Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
92  TUKind(TUKind), SkipMainFilePreamble(0, true),
93  CurSubmoduleState(&NullSubmoduleState) {
94  OwnsHeaderSearch = OwnsHeaders;
95 
96  // Default to discarding comments.
97  KeepComments = false;
98  KeepMacroComments = false;
99  SuppressIncludeNotFoundError = false;
100 
101  // Macro expansion is enabled.
102  DisableMacroExpansion = false;
103  MacroExpansionInDirectivesOverride = false;
104  InMacroArgs = false;
105  ArgMacro = nullptr;
106  InMacroArgPreExpansion = false;
107  NumCachedTokenLexers = 0;
108  PragmasEnabled = true;
109  ParsingIfOrElifDirective = false;
110  PreprocessedOutput = false;
111 
112  // We haven't read anything from the external source.
113  ReadMacrosFromExternalSource = false;
114 
115  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
116  // a macro. They get unpoisoned where it is allowed.
117  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
118  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
119  if (getLangOpts().CPlusPlus2a) {
120  (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
121  SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
122  } else {
123  Ident__VA_OPT__ = nullptr;
124  }
125 
126  // Initialize the pragma handlers.
127  RegisterBuiltinPragmas();
128 
129  // Initialize builtin macros like __LINE__ and friends.
130  RegisterBuiltinMacros();
131 
132  if(LangOpts.Borland) {
133  Ident__exception_info = getIdentifierInfo("_exception_info");
134  Ident___exception_info = getIdentifierInfo("__exception_info");
135  Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
136  Ident__exception_code = getIdentifierInfo("_exception_code");
137  Ident___exception_code = getIdentifierInfo("__exception_code");
138  Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
139  Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
140  Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
141  Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
142  } else {
143  Ident__exception_info = Ident__exception_code = nullptr;
144  Ident__abnormal_termination = Ident___exception_info = nullptr;
145  Ident___exception_code = Ident___abnormal_termination = nullptr;
146  Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
147  Ident_AbnormalTermination = nullptr;
148  }
149 
150  // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
152  SkippingUntilPragmaHdrStop = true;
153 
154  // If using a PCH with a through header, start skipping tokens.
155  if (!this->PPOpts->PCHThroughHeader.empty() &&
156  !this->PPOpts->ImplicitPCHInclude.empty())
157  SkippingUntilPCHThroughHeader = true;
158 
159  if (this->PPOpts->GeneratePreamble)
160  PreambleConditionalStack.startRecording();
161 
162  ExcludedConditionalDirectiveSkipMappings =
163  this->PPOpts->ExcludedConditionalDirectiveSkipMappings;
164  if (ExcludedConditionalDirectiveSkipMappings)
165  ExcludedConditionalDirectiveSkipMappings->clear();
166 }
167 
169  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
170 
171  IncludeMacroStack.clear();
172 
173  // Destroy any macro definitions.
174  while (MacroInfoChain *I = MIChainHead) {
175  MIChainHead = I->Next;
176  I->~MacroInfoChain();
177  }
178 
179  // Free any cached macro expanders.
180  // This populates MacroArgCache, so all TokenLexers need to be destroyed
181  // before the code below that frees up the MacroArgCache list.
182  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
183  CurTokenLexer.reset();
184 
185  // Free any cached MacroArgs.
186  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
187  ArgList = ArgList->deallocate();
188 
189  // Delete the header search info, if we own it.
190  if (OwnsHeaderSearch)
191  delete &HeaderInfo;
192 }
193 
195  const TargetInfo *AuxTarget) {
196  assert((!this->Target || this->Target == &Target) &&
197  "Invalid override of target information");
198  this->Target = &Target;
199 
200  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
201  "Invalid override of aux target information.");
202  this->AuxTarget = AuxTarget;
203 
204  // Initialize information about built-ins.
205  BuiltinInfo.InitializeTarget(Target, AuxTarget);
206  HeaderInfo.setTarget(Target);
207 
208  // Populate the identifier table with info about keywords for the current language.
209  Identifiers.AddKeywords(LangOpts);
210 }
211 
213  NumEnteredSourceFiles = 0;
214 
215  // Reset pragmas
216  PragmaHandlersBackup = std::move(PragmaHandlers);
217  PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
218  RegisterBuiltinPragmas();
219 
220  // Reset PredefinesFileID
221  PredefinesFileID = FileID();
222 }
223 
225  NumEnteredSourceFiles = 1;
226 
227  PragmaHandlers = std::move(PragmaHandlersBackup);
228 }
229 
230 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
231  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
232  << getSpelling(Tok) << "'";
233 
234  if (!DumpFlags) return;
235 
236  llvm::errs() << "\t";
237  if (Tok.isAtStartOfLine())
238  llvm::errs() << " [StartOfLine]";
239  if (Tok.hasLeadingSpace())
240  llvm::errs() << " [LeadingSpace]";
241  if (Tok.isExpandDisabled())
242  llvm::errs() << " [ExpandDisabled]";
243  if (Tok.needsCleaning()) {
244  const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
245  llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
246  << "']";
247  }
248 
249  llvm::errs() << "\tLoc=<";
250  DumpLocation(Tok.getLocation());
251  llvm::errs() << ">";
252 }
253 
255  Loc.print(llvm::errs(), SourceMgr);
256 }
257 
258 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
259  llvm::errs() << "MACRO: ";
260  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
262  llvm::errs() << " ";
263  }
264  llvm::errs() << "\n";
265 }
266 
268  llvm::errs() << "\n*** Preprocessor Stats:\n";
269  llvm::errs() << NumDirectives << " directives found:\n";
270  llvm::errs() << " " << NumDefined << " #define.\n";
271  llvm::errs() << " " << NumUndefined << " #undef.\n";
272  llvm::errs() << " #include/#include_next/#import:\n";
273  llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
274  llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
275  llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
276  llvm::errs() << " " << NumElse << " #else/#elif.\n";
277  llvm::errs() << " " << NumEndif << " #endif.\n";
278  llvm::errs() << " " << NumPragma << " #pragma.\n";
279  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
280 
281  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
282  << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
283  << NumFastMacroExpanded << " on the fast path.\n";
284  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
285  << " token paste (##) operations performed, "
286  << NumFastTokenPaste << " on the fast path.\n";
287 
288  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
289 
290  llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
291  llvm::errs() << "\n Macro Expanded Tokens: "
292  << llvm::capacity_in_bytes(MacroExpandedTokens);
293  llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
294  // FIXME: List information for all submodules.
295  llvm::errs() << "\n Macros: "
296  << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
297  llvm::errs() << "\n #pragma push_macro Info: "
298  << llvm::capacity_in_bytes(PragmaPushMacroInfo);
299  llvm::errs() << "\n Poison Reasons: "
300  << llvm::capacity_in_bytes(PoisonReasons);
301  llvm::errs() << "\n Comment Handlers: "
302  << llvm::capacity_in_bytes(CommentHandlers) << "\n";
303 }
304 
306 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
307  if (IncludeExternalMacros && ExternalSource &&
308  !ReadMacrosFromExternalSource) {
309  ReadMacrosFromExternalSource = true;
310  ExternalSource->ReadDefinedMacros();
311  }
312 
313  // Make sure we cover all macros in visible modules.
314  for (const ModuleMacro &Macro : ModuleMacros)
315  CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
316 
317  return CurSubmoduleState->Macros.begin();
318 }
319 
321  return BP.getTotalMemory()
322  + llvm::capacity_in_bytes(MacroExpandedTokens)
323  + Predefines.capacity() /* Predefines buffer. */
324  // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
325  // and ModuleMacros.
326  + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
327  + llvm::capacity_in_bytes(PragmaPushMacroInfo)
328  + llvm::capacity_in_bytes(PoisonReasons)
329  + llvm::capacity_in_bytes(CommentHandlers);
330 }
331 
333 Preprocessor::macro_end(bool IncludeExternalMacros) const {
334  if (IncludeExternalMacros && ExternalSource &&
335  !ReadMacrosFromExternalSource) {
336  ReadMacrosFromExternalSource = true;
337  ExternalSource->ReadDefinedMacros();
338  }
339 
340  return CurSubmoduleState->Macros.end();
341 }
342 
343 /// Compares macro tokens with a specified token value sequence.
344 static bool MacroDefinitionEquals(const MacroInfo *MI,
345  ArrayRef<TokenValue> Tokens) {
346  return Tokens.size() == MI->getNumTokens() &&
347  std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
348 }
349 
351  SourceLocation Loc,
352  ArrayRef<TokenValue> Tokens) const {
353  SourceLocation BestLocation;
354  StringRef BestSpelling;
356  I != E; ++I) {
358  Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
359  if (!Def || !Def.getMacroInfo())
360  continue;
361  if (!Def.getMacroInfo()->isObjectLike())
362  continue;
363  if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
364  continue;
365  SourceLocation Location = Def.getLocation();
366  // Choose the macro defined latest.
367  if (BestLocation.isInvalid() ||
368  (Location.isValid() &&
369  SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
370  BestLocation = Location;
371  BestSpelling = I->first->getName();
372  }
373  }
374  return BestSpelling;
375 }
376 
378  if (CurLexer)
379  CurLexerKind = CLK_Lexer;
380  else if (CurTokenLexer)
381  CurLexerKind = CLK_TokenLexer;
382  else
383  CurLexerKind = CLK_CachingLexer;
384 }
385 
387  unsigned CompleteLine,
388  unsigned CompleteColumn) {
389  assert(File);
390  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
391  assert(!CodeCompletionFile && "Already set");
392 
393  using llvm::MemoryBuffer;
394 
395  // Load the actual file's contents.
396  bool Invalid = false;
397  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
398  if (Invalid)
399  return true;
400 
401  // Find the byte position of the truncation point.
402  const char *Position = Buffer->getBufferStart();
403  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
404  for (; *Position; ++Position) {
405  if (*Position != '\r' && *Position != '\n')
406  continue;
407 
408  // Eat \r\n or \n\r as a single line.
409  if ((Position[1] == '\r' || Position[1] == '\n') &&
410  Position[0] != Position[1])
411  ++Position;
412  ++Position;
413  break;
414  }
415  }
416 
417  Position += CompleteColumn - 1;
418 
419  // If pointing inside the preamble, adjust the position at the beginning of
420  // the file after the preamble.
421  if (SkipMainFilePreamble.first &&
422  SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
423  if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
424  Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
425  }
426 
427  if (Position > Buffer->getBufferEnd())
428  Position = Buffer->getBufferEnd();
429 
430  CodeCompletionFile = File;
431  CodeCompletionOffset = Position - Buffer->getBufferStart();
432 
433  auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
434  Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
435  char *NewBuf = NewBuffer->getBufferStart();
436  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
437  *NewPos = '\0';
438  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
439  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
440 
441  return false;
442 }
443 
444 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
445  bool IsAngled) {
446  if (CodeComplete)
447  CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
449 }
450 
452  if (CodeComplete)
453  CodeComplete->CodeCompleteNaturalLanguage();
455 }
456 
457 /// getSpelling - This method is used to get the spelling of a token into a
458 /// SmallVector. Note that the returned StringRef may not point to the
459 /// supplied buffer if a copy can be avoided.
461  SmallVectorImpl<char> &Buffer,
462  bool *Invalid) const {
463  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
464  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
465  // Try the fast path.
466  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
467  return II->getName();
468  }
469 
470  // Resize the buffer if we need to copy into it.
471  if (Tok.needsCleaning())
472  Buffer.resize(Tok.getLength());
473 
474  const char *Ptr = Buffer.data();
475  unsigned Len = getSpelling(Tok, Ptr, Invalid);
476  return StringRef(Ptr, Len);
477 }
478 
479 /// CreateString - Plop the specified string into a scratch buffer and return a
480 /// location for it. If specified, the source location provides a source
481 /// location for the token.
482 void Preprocessor::CreateString(StringRef Str, Token &Tok,
483  SourceLocation ExpansionLocStart,
484  SourceLocation ExpansionLocEnd) {
485  Tok.setLength(Str.size());
486 
487  const char *DestPtr;
488  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
489 
490  if (ExpansionLocStart.isValid())
491  Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
492  ExpansionLocEnd, Str.size());
493  Tok.setLocation(Loc);
494 
495  // If this is a raw identifier or a literal token, set the pointer data.
496  if (Tok.is(tok::raw_identifier))
497  Tok.setRawIdentifierData(DestPtr);
498  else if (Tok.isLiteral())
499  Tok.setLiteralData(DestPtr);
500 }
501 
503  auto &SM = getSourceManager();
504  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
505  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
506  bool Invalid = false;
507  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
508  if (Invalid)
509  return SourceLocation();
510 
511  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
512  const char *DestPtr;
513  SourceLocation Spelling =
514  ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
515  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
516 }
517 
519  if (!getLangOpts().isCompilingModule())
520  return nullptr;
521 
522  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
523 }
524 
525 //===----------------------------------------------------------------------===//
526 // Preprocessor Initialization Methods
527 //===----------------------------------------------------------------------===//
528 
529 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
530 /// which implicitly adds the builtin defines etc.
532  // We do not allow the preprocessor to reenter the main file. Doing so will
533  // cause FileID's to accumulate information from both runs (e.g. #line
534  // information) and predefined macros aren't guaranteed to be set properly.
535  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
536  FileID MainFileID = SourceMgr.getMainFileID();
537 
538  // If MainFileID is loaded it means we loaded an AST file, no need to enter
539  // a main file.
540  if (!SourceMgr.isLoadedFileID(MainFileID)) {
541  // Enter the main file source buffer.
542  EnterSourceFile(MainFileID, nullptr, SourceLocation());
543 
544  // If we've been asked to skip bytes in the main file (e.g., as part of a
545  // precompiled preamble), do so now.
546  if (SkipMainFilePreamble.first > 0)
547  CurLexer->SetByteOffset(SkipMainFilePreamble.first,
548  SkipMainFilePreamble.second);
549 
550  // Tell the header info that the main file was entered. If the file is later
551  // #imported, it won't be re-entered.
552  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
553  HeaderInfo.IncrementIncludeCount(FE);
554  }
555 
556  // Preprocess Predefines to populate the initial preprocessor state.
557  std::unique_ptr<llvm::MemoryBuffer> SB =
558  llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
559  assert(SB && "Cannot create predefined source buffer");
560  FileID FID = SourceMgr.createFileID(std::move(SB));
561  assert(FID.isValid() && "Could not create FileID for predefines?");
562  setPredefinesFileID(FID);
563 
564  // Start parsing the predefines.
565  EnterSourceFile(FID, nullptr, SourceLocation());
566 
567  if (!PPOpts->PCHThroughHeader.empty()) {
568  // Lookup and save the FileID for the through header. If it isn't found
569  // in the search path, it's a fatal error.
570  const DirectoryLookup *CurDir;
572  SourceLocation(), PPOpts->PCHThroughHeader,
573  /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, CurDir,
574  /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
575  /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
576  /*IsFrameworkFound=*/nullptr);
577  if (!File) {
578  Diag(SourceLocation(), diag::err_pp_through_header_not_found)
579  << PPOpts->PCHThroughHeader;
580  return;
581  }
582  setPCHThroughHeaderFileID(
583  SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User));
584  }
585 
586  // Skip tokens from the Predefines and if needed the main file.
587  if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
588  (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
590 }
591 
592 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
593  assert(PCHThroughHeaderFileID.isInvalid() &&
594  "PCHThroughHeaderFileID already set!");
595  PCHThroughHeaderFileID = FID;
596 }
597 
599  assert(PCHThroughHeaderFileID.isValid() &&
600  "Invalid PCH through header FileID");
601  return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
602 }
603 
605  return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
606  PCHThroughHeaderFileID.isValid();
607 }
608 
610  return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
611  PCHThroughHeaderFileID.isValid();
612 }
613 
615  return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
616 }
617 
619  return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
620 }
621 
622 /// Skip tokens until after the #include of the through header or
623 /// until after a #pragma hdrstop is seen. Tokens in the predefines file
624 /// and the main file may be skipped. If the end of the predefines file
625 /// is reached, skipping continues into the main file. If the end of the
626 /// main file is reached, it's a fatal error.
628  bool ReachedMainFileEOF = false;
629  bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
630  bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
631  Token Tok;
632  while (true) {
633  bool InPredefines =
634  (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
635  switch (CurLexerKind) {
636  case CLK_Lexer:
637  CurLexer->Lex(Tok);
638  break;
639  case CLK_TokenLexer:
640  CurTokenLexer->Lex(Tok);
641  break;
642  case CLK_CachingLexer:
643  CachingLex(Tok);
644  break;
645  case CLK_LexAfterModuleImport:
647  break;
648  }
649  if (Tok.is(tok::eof) && !InPredefines) {
650  ReachedMainFileEOF = true;
651  break;
652  }
653  if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
654  break;
655  if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
656  break;
657  }
658  if (ReachedMainFileEOF) {
659  if (UsingPCHThroughHeader)
660  Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
661  << PPOpts->PCHThroughHeader << 1;
662  else if (!PPOpts->PCHWithHdrStopCreate)
663  Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
664  }
665 }
666 
667 void Preprocessor::replayPreambleConditionalStack() {
668  // Restore the conditional stack from the preamble, if there is one.
669  if (PreambleConditionalStack.isReplaying()) {
670  assert(CurPPLexer &&
671  "CurPPLexer is null when calling replayPreambleConditionalStack.");
672  CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
673  PreambleConditionalStack.doneReplaying();
674  if (PreambleConditionalStack.reachedEOFWhileSkipping())
675  SkipExcludedConditionalBlock(
676  PreambleConditionalStack.SkipInfo->HashTokenLoc,
677  PreambleConditionalStack.SkipInfo->IfTokenLoc,
678  PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
679  PreambleConditionalStack.SkipInfo->FoundElse,
680  PreambleConditionalStack.SkipInfo->ElseLoc);
681  }
682 }
683 
685  // Notify the client that we reached the end of the source file.
686  if (Callbacks)
687  Callbacks->EndOfMainFile();
688 }
689 
690 //===----------------------------------------------------------------------===//
691 // Lexer Event Handling.
692 //===----------------------------------------------------------------------===//
693 
694 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
695 /// identifier information for the token and install it into the token,
696 /// updating the token kind accordingly.
698  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
699 
700  // Look up this token, see if it is a macro, or if it is a language keyword.
701  IdentifierInfo *II;
702  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
703  // No cleaning needed, just use the characters from the lexed buffer.
704  II = getIdentifierInfo(Identifier.getRawIdentifier());
705  } else {
706  // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
707  SmallString<64> IdentifierBuffer;
708  StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
709 
710  if (Identifier.hasUCN()) {
711  SmallString<64> UCNIdentifierBuffer;
712  expandUCNs(UCNIdentifierBuffer, CleanedStr);
713  II = getIdentifierInfo(UCNIdentifierBuffer);
714  } else {
715  II = getIdentifierInfo(CleanedStr);
716  }
717  }
718 
719  // Update the token info (identifier info and appropriate token kind).
720  Identifier.setIdentifierInfo(II);
721  if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
723  Identifier.setKind(tok::identifier);
724  else
725  Identifier.setKind(II->getTokenID());
726 
727  return II;
728 }
729 
730 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
731  PoisonReasons[II] = DiagID;
732 }
733 
735  assert(Ident__exception_code && Ident__exception_info);
736  assert(Ident___exception_code && Ident___exception_info);
737  Ident__exception_code->setIsPoisoned(Poison);
738  Ident___exception_code->setIsPoisoned(Poison);
739  Ident_GetExceptionCode->setIsPoisoned(Poison);
740  Ident__exception_info->setIsPoisoned(Poison);
741  Ident___exception_info->setIsPoisoned(Poison);
742  Ident_GetExceptionInfo->setIsPoisoned(Poison);
743  Ident__abnormal_termination->setIsPoisoned(Poison);
744  Ident___abnormal_termination->setIsPoisoned(Poison);
745  Ident_AbnormalTermination->setIsPoisoned(Poison);
746 }
747 
749  assert(Identifier.getIdentifierInfo() &&
750  "Can't handle identifiers without identifier info!");
751  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
752  PoisonReasons.find(Identifier.getIdentifierInfo());
753  if(it == PoisonReasons.end())
754  Diag(Identifier, diag::err_pp_used_poisoned_id);
755  else
756  Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
757 }
758 
759 /// Returns a diagnostic message kind for reporting a future keyword as
760 /// appropriate for the identifier and specified language.
762  const LangOptions &LangOpts) {
763  assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
764 
765  if (LangOpts.CPlusPlus)
766  return llvm::StringSwitch<diag::kind>(II.getName())
767 #define CXX11_KEYWORD(NAME, FLAGS) \
768  .Case(#NAME, diag::warn_cxx11_keyword)
769 #define CXX2A_KEYWORD(NAME, FLAGS) \
770  .Case(#NAME, diag::warn_cxx2a_keyword)
771 #include "clang/Basic/TokenKinds.def"
772  ;
773 
774  llvm_unreachable(
775  "Keyword not known to come from a newer Standard or proposed Standard");
776 }
777 
778 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
779  assert(II.isOutOfDate() && "not out of date");
781 }
782 
783 /// HandleIdentifier - This callback is invoked when the lexer reads an
784 /// identifier. This callback looks up the identifier in the map and/or
785 /// potentially macro expands it or turns it into a named token (like 'for').
786 ///
787 /// Note that callers of this method are guarded by checking the
788 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
789 /// IdentifierInfo methods that compute these properties will need to change to
790 /// match.
792  assert(Identifier.getIdentifierInfo() &&
793  "Can't handle identifiers without identifier info!");
794 
795  IdentifierInfo &II = *Identifier.getIdentifierInfo();
796 
797  // If the information about this identifier is out of date, update it from
798  // the external source.
799  // We have to treat __VA_ARGS__ in a special way, since it gets
800  // serialized with isPoisoned = true, but our preprocessor may have
801  // unpoisoned it if we're defining a C99 macro.
802  if (II.isOutOfDate()) {
803  bool CurrentIsPoisoned = false;
804  const bool IsSpecialVariadicMacro =
805  &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
806  if (IsSpecialVariadicMacro)
807  CurrentIsPoisoned = II.isPoisoned();
808 
809  updateOutOfDateIdentifier(II);
810  Identifier.setKind(II.getTokenID());
811 
812  if (IsSpecialVariadicMacro)
813  II.setIsPoisoned(CurrentIsPoisoned);
814  }
815 
816  // If this identifier was poisoned, and if it was not produced from a macro
817  // expansion, emit an error.
818  if (II.isPoisoned() && CurPPLexer) {
819  HandlePoisonedIdentifier(Identifier);
820  }
821 
822  // If this is a macro to be expanded, do it.
823  if (MacroDefinition MD = getMacroDefinition(&II)) {
824  auto *MI = MD.getMacroInfo();
825  assert(MI && "macro definition with no macro info?");
826  if (!DisableMacroExpansion) {
827  if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
828  // C99 6.10.3p10: If the preprocessing token immediately after the
829  // macro name isn't a '(', this macro should not be expanded.
830  if (!MI->isFunctionLike() || isNextPPTokenLParen())
831  return HandleMacroExpandedIdentifier(Identifier, MD);
832  } else {
833  // C99 6.10.3.4p2 says that a disabled macro may never again be
834  // expanded, even if it's in a context where it could be expanded in the
835  // future.
836  Identifier.setFlag(Token::DisableExpand);
837  if (MI->isObjectLike() || isNextPPTokenLParen())
838  Diag(Identifier, diag::pp_disabled_macro_expansion);
839  }
840  }
841  }
842 
843  // If this identifier is a keyword in a newer Standard or proposed Standard,
844  // produce a warning. Don't warn if we're not considering macro expansion,
845  // since this identifier might be the name of a macro.
846  // FIXME: This warning is disabled in cases where it shouldn't be, like
847  // "#define constexpr constexpr", "int constexpr;"
848  if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
849  Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
850  << II.getName();
851  // Don't diagnose this keyword again in this translation unit.
852  II.setIsFutureCompatKeyword(false);
853  }
854 
855  // If this is an extension token, diagnose its use.
856  // We avoid diagnosing tokens that originate from macro definitions.
857  // FIXME: This warning is disabled in cases where it shouldn't be,
858  // like "#define TY typeof", "TY(1) x".
859  if (II.isExtensionToken() && !DisableMacroExpansion)
860  Diag(Identifier, diag::ext_token_used);
861 
862  // If this is the 'import' contextual keyword following an '@', note
863  // that the next token indicates a module name.
864  //
865  // Note that we do not treat 'import' as a contextual
866  // keyword when we're in a caching lexer, because caching lexers only get
867  // used in contexts where import declarations are disallowed.
868  //
869  // Likewise if this is the C++ Modules TS import keyword.
870  if (((LastTokenWasAt && II.isModulesImport()) ||
871  Identifier.is(tok::kw_import)) &&
872  !InMacroArgs && !DisableMacroExpansion &&
873  (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
874  CurLexerKind != CLK_CachingLexer) {
875  ModuleImportLoc = Identifier.getLocation();
876  ModuleImportPath.clear();
877  ModuleImportExpectsIdentifier = true;
878  CurLexerKind = CLK_LexAfterModuleImport;
879  }
880  return true;
881 }
882 
884  ++LexLevel;
885 
886  // We loop here until a lex function returns a token; this avoids recursion.
887  bool ReturnedToken;
888  do {
889  switch (CurLexerKind) {
890  case CLK_Lexer:
891  ReturnedToken = CurLexer->Lex(Result);
892  break;
893  case CLK_TokenLexer:
894  ReturnedToken = CurTokenLexer->Lex(Result);
895  break;
896  case CLK_CachingLexer:
897  CachingLex(Result);
898  ReturnedToken = true;
899  break;
900  case CLK_LexAfterModuleImport:
901  ReturnedToken = LexAfterModuleImport(Result);
902  break;
903  }
904  } while (!ReturnedToken);
905 
906  if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
907  // Remember the identifier before code completion token.
910  // Set IdenfitierInfo to null to avoid confusing code that handles both
911  // identifiers and completion tokens.
912  Result.setIdentifierInfo(nullptr);
913  }
914 
915  // Update ImportSeqState to track our position within a C++20 import-seq
916  // if this token is being produced as a result of phase 4 of translation.
917  if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
918  !Result.getFlag(Token::IsReinjected)) {
919  switch (Result.getKind()) {
920  case tok::l_paren: case tok::l_square: case tok::l_brace:
921  ImportSeqState.handleOpenBracket();
922  break;
923  case tok::r_paren: case tok::r_square:
924  ImportSeqState.handleCloseBracket();
925  break;
926  case tok::r_brace:
927  ImportSeqState.handleCloseBrace();
928  break;
929  case tok::semi:
930  ImportSeqState.handleSemi();
931  break;
932  case tok::header_name:
933  case tok::annot_header_unit:
934  ImportSeqState.handleHeaderName();
935  break;
936  case tok::kw_export:
937  ImportSeqState.handleExport();
938  break;
939  case tok::identifier:
940  if (Result.getIdentifierInfo()->isModulesImport()) {
941  ImportSeqState.handleImport();
942  if (ImportSeqState.afterImportSeq()) {
943  ModuleImportLoc = Result.getLocation();
944  ModuleImportPath.clear();
945  ModuleImportExpectsIdentifier = true;
946  CurLexerKind = CLK_LexAfterModuleImport;
947  }
948  break;
949  }
950  LLVM_FALLTHROUGH;
951  default:
952  ImportSeqState.handleMisc();
953  break;
954  }
955  }
956 
957  LastTokenWasAt = Result.is(tok::at);
958  --LexLevel;
959  if (OnToken && LexLevel == 0 && !Result.getFlag(Token::IsReinjected))
960  OnToken(Result);
961 }
962 
963 /// Lex a header-name token (including one formed from header-name-tokens if
964 /// \p AllowConcatenation is \c true).
965 ///
966 /// \param FilenameTok Filled in with the next token. On success, this will
967 /// be either a header_name token. On failure, it will be whatever other
968 /// token was found instead.
969 /// \param AllowMacroExpansion If \c true, allow the header name to be formed
970 /// by macro expansion (concatenating tokens as necessary if the first
971 /// token is a '<').
972 /// \return \c true if we reached EOD or EOF while looking for a > token in
973 /// a concatenated header name and diagnosed it. \c false otherwise.
974 bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
975  // Lex using header-name tokenization rules if tokens are being lexed from
976  // a file. Just grab a token normally if we're in a macro expansion.
977  if (CurPPLexer)
978  CurPPLexer->LexIncludeFilename(FilenameTok);
979  else
980  Lex(FilenameTok);
981 
982  // This could be a <foo/bar.h> file coming from a macro expansion. In this
983  // case, glue the tokens together into an angle_string_literal token.
984  SmallString<128> FilenameBuffer;
985  if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
986  bool StartOfLine = FilenameTok.isAtStartOfLine();
987  bool LeadingSpace = FilenameTok.hasLeadingSpace();
988  bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
989 
990  SourceLocation Start = FilenameTok.getLocation();
992  FilenameBuffer.push_back('<');
993 
994  // Consume tokens until we find a '>'.
995  // FIXME: A header-name could be formed starting or ending with an
996  // alternative token. It's not clear whether that's ill-formed in all
997  // cases.
998  while (FilenameTok.isNot(tok::greater)) {
999  Lex(FilenameTok);
1000  if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
1001  Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
1002  Diag(Start, diag::note_matching) << tok::less;
1003  return true;
1004  }
1005 
1006  End = FilenameTok.getLocation();
1007 
1008  // FIXME: Provide code completion for #includes.
1009  if (FilenameTok.is(tok::code_completion)) {
1011  Lex(FilenameTok);
1012  continue;
1013  }
1014 
1015  // Append the spelling of this token to the buffer. If there was a space
1016  // before it, add it now.
1017  if (FilenameTok.hasLeadingSpace())
1018  FilenameBuffer.push_back(' ');
1019 
1020  // Get the spelling of the token, directly into FilenameBuffer if
1021  // possible.
1022  size_t PreAppendSize = FilenameBuffer.size();
1023  FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
1024 
1025  const char *BufPtr = &FilenameBuffer[PreAppendSize];
1026  unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
1027 
1028  // If the token was spelled somewhere else, copy it into FilenameBuffer.
1029  if (BufPtr != &FilenameBuffer[PreAppendSize])
1030  memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1031 
1032  // Resize FilenameBuffer to the correct size.
1033  if (FilenameTok.getLength() != ActualLen)
1034  FilenameBuffer.resize(PreAppendSize + ActualLen);
1035  }
1036 
1037  FilenameTok.startToken();
1038  FilenameTok.setKind(tok::header_name);
1039  FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine);
1040  FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace);
1041  FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro);
1042  CreateString(FilenameBuffer, FilenameTok, Start, End);
1043  } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
1044  // Convert a string-literal token of the form " h-char-sequence "
1045  // (produced by macro expansion) into a header-name token.
1046  //
1047  // The rules for header-names don't quite match the rules for
1048  // string-literals, but all the places where they differ result in
1049  // undefined behavior, so we can and do treat them the same.
1050  //
1051  // A string-literal with a prefix or suffix is not translated into a
1052  // header-name. This could theoretically be observable via the C++20
1053  // context-sensitive header-name formation rules.
1054  StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
1055  if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1056  FilenameTok.setKind(tok::header_name);
1057  }
1058 
1059  return false;
1060 }
1061 
1062 /// Collect the tokens of a C++20 pp-import-suffix.
1064  // FIXME: For error recovery, consider recognizing attribute syntax here
1065  // and terminating / diagnosing a missing semicolon if we find anything
1066  // else? (Can we leave that to the parser?)
1067  unsigned BracketDepth = 0;
1068  while (true) {
1069  Toks.emplace_back();
1070  Lex(Toks.back());
1071 
1072  switch (Toks.back().getKind()) {
1073  case tok::l_paren: case tok::l_square: case tok::l_brace:
1074  ++BracketDepth;
1075  break;
1076 
1077  case tok::r_paren: case tok::r_square: case tok::r_brace:
1078  if (BracketDepth == 0)
1079  return;
1080  --BracketDepth;
1081  break;
1082 
1083  case tok::semi:
1084  if (BracketDepth == 0)
1085  return;
1086  break;
1087 
1088  case tok::eof:
1089  return;
1090 
1091  default:
1092  break;
1093  }
1094  }
1095 }
1096 
1097 
1098 /// Lex a token following the 'import' contextual keyword.
1099 ///
1100 /// pp-import: [C++20]
1101 /// import header-name pp-import-suffix[opt] ;
1102 /// import header-name-tokens pp-import-suffix[opt] ;
1103 /// [ObjC] @ import module-name ;
1104 /// [Clang] import module-name ;
1105 ///
1106 /// header-name-tokens:
1107 /// string-literal
1108 /// < [any sequence of preprocessing-tokens other than >] >
1109 ///
1110 /// module-name:
1111 /// module-name-qualifier[opt] identifier
1112 ///
1113 /// module-name-qualifier
1114 /// module-name-qualifier[opt] identifier .
1115 ///
1116 /// We respond to a pp-import by importing macros from the named module.
1118  // Figure out what kind of lexer we actually have.
1120 
1121  // Lex the next token. The header-name lexing rules are used at the start of
1122  // a pp-import.
1123  //
1124  // For now, we only support header-name imports in C++20 mode.
1125  // FIXME: Should we allow this in all language modes that support an import
1126  // declaration as an extension?
1127  if (ModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1128  if (LexHeaderName(Result))
1129  return true;
1130  } else {
1131  Lex(Result);
1132  }
1133 
1134  // Allocate a holding buffer for a sequence of tokens and introduce it into
1135  // the token stream.
1136  auto EnterTokens = [this](ArrayRef<Token> Toks) {
1137  auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1138  std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1139  EnterTokenStream(std::move(ToksCopy), Toks.size(),
1140  /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1141  };
1142 
1143  // Check for a header-name.
1144  SmallVector<Token, 32> Suffix;
1145  if (Result.is(tok::header_name)) {
1146  // Enter the header-name token into the token stream; a Lex action cannot
1147  // both return a token and cache tokens (doing so would corrupt the token
1148  // cache if the call to Lex comes from CachingLex / PeekAhead).
1149  Suffix.push_back(Result);
1150 
1151  // Consume the pp-import-suffix and expand any macros in it now. We'll add
1152  // it back into the token stream later.
1153  CollectPpImportSuffix(Suffix);
1154  if (Suffix.back().isNot(tok::semi)) {
1155  // This is not a pp-import after all.
1156  EnterTokens(Suffix);
1157  return false;
1158  }
1159 
1160  // C++2a [cpp.module]p1:
1161  // The ';' preprocessing-token terminating a pp-import shall not have
1162  // been produced by macro replacement.
1163  SourceLocation SemiLoc = Suffix.back().getLocation();
1164  if (SemiLoc.isMacroID())
1165  Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1166 
1167  // Reconstitute the import token.
1168  Token ImportTok;
1169  ImportTok.startToken();
1170  ImportTok.setKind(tok::kw_import);
1171  ImportTok.setLocation(ModuleImportLoc);
1172  ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
1173  ImportTok.setLength(6);
1174 
1175  auto Action = HandleHeaderIncludeOrImport(
1176  /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
1177  switch (Action.Kind) {
1178  case ImportAction::None:
1179  break;
1180 
1181  case ImportAction::ModuleBegin:
1182  // Let the parser know we're textually entering the module.
1183  Suffix.emplace_back();
1184  Suffix.back().startToken();
1185  Suffix.back().setKind(tok::annot_module_begin);
1186  Suffix.back().setLocation(SemiLoc);
1187  Suffix.back().setAnnotationEndLoc(SemiLoc);
1188  Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1189  LLVM_FALLTHROUGH;
1190 
1191  case ImportAction::ModuleImport:
1192  case ImportAction::SkippedModuleImport:
1193  // We chose to import (or textually enter) the file. Convert the
1194  // header-name token into a header unit annotation token.
1195  Suffix[0].setKind(tok::annot_header_unit);
1196  Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1197  Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1198  // FIXME: Call the moduleImport callback?
1199  break;
1200  }
1201 
1202  EnterTokens(Suffix);
1203  return false;
1204  }
1205 
1206  // The token sequence
1207  //
1208  // import identifier (. identifier)*
1209  //
1210  // indicates a module import directive. We already saw the 'import'
1211  // contextual keyword, so now we're looking for the identifiers.
1212  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1213  // We expected to see an identifier here, and we did; continue handling
1214  // identifiers.
1215  ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
1216  Result.getLocation()));
1217  ModuleImportExpectsIdentifier = false;
1218  CurLexerKind = CLK_LexAfterModuleImport;
1219  return true;
1220  }
1221 
1222  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1223  // see the next identifier. (We can also see a '[[' that begins an
1224  // attribute-specifier-seq here under the C++ Modules TS.)
1225  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1226  ModuleImportExpectsIdentifier = true;
1227  CurLexerKind = CLK_LexAfterModuleImport;
1228  return true;
1229  }
1230 
1231  // If we didn't recognize a module name at all, this is not a (valid) import.
1232  if (ModuleImportPath.empty() || Result.is(tok::eof))
1233  return true;
1234 
1235  // Consume the pp-import-suffix and expand any macros in it now, if we're not
1236  // at the semicolon already.
1237  SourceLocation SemiLoc = Result.getLocation();
1238  if (Result.isNot(tok::semi)) {
1239  Suffix.push_back(Result);
1240  CollectPpImportSuffix(Suffix);
1241  if (Suffix.back().isNot(tok::semi)) {
1242  // This is not an import after all.
1243  EnterTokens(Suffix);
1244  return false;
1245  }
1246  SemiLoc = Suffix.back().getLocation();
1247  }
1248 
1249  // Under the Modules TS, the dot is just part of the module name, and not
1250  // a real hierarchy separator. Flatten such module names now.
1251  //
1252  // FIXME: Is this the right level to be performing this transformation?
1253  std::string FlatModuleName;
1254  if (getLangOpts().ModulesTS || getLangOpts().CPlusPlusModules) {
1255  for (auto &Piece : ModuleImportPath) {
1256  if (!FlatModuleName.empty())
1257  FlatModuleName += ".";
1258  FlatModuleName += Piece.first->getName();
1259  }
1260  SourceLocation FirstPathLoc = ModuleImportPath[0].second;
1261  ModuleImportPath.clear();
1262  ModuleImportPath.push_back(
1263  std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1264  }
1265 
1266  Module *Imported = nullptr;
1267  if (getLangOpts().Modules) {
1268  Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1269  ModuleImportPath,
1271  /*IsInclusionDirective=*/false);
1272  if (Imported)
1273  makeModuleVisible(Imported, SemiLoc);
1274  }
1275  if (Callbacks)
1276  Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1277 
1278  if (!Suffix.empty()) {
1279  EnterTokens(Suffix);
1280  return false;
1281  }
1282  return true;
1283 }
1284 
1286  CurSubmoduleState->VisibleModules.setVisible(
1287  M, Loc, [](Module *) {},
1288  [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1289  // FIXME: Include the path in the diagnostic.
1290  // FIXME: Include the import location for the conflicting module.
1291  Diag(ModuleImportLoc, diag::warn_module_conflict)
1292  << Path[0]->getFullModuleName()
1293  << Conflict->getFullModuleName()
1294  << Message;
1295  });
1296 
1297  // Add this module to the imports list of the currently-built submodule.
1298  if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1299  BuildingSubmoduleStack.back().M->Imports.insert(M);
1300 }
1301 
1303  const char *DiagnosticTag,
1304  bool AllowMacroExpansion) {
1305  // We need at least one string literal.
1306  if (Result.isNot(tok::string_literal)) {
1307  Diag(Result, diag::err_expected_string_literal)
1308  << /*Source='in...'*/0 << DiagnosticTag;
1309  return false;
1310  }
1311 
1312  // Lex string literal tokens, optionally with macro expansion.
1313  SmallVector<Token, 4> StrToks;
1314  do {
1315  StrToks.push_back(Result);
1316 
1317  if (Result.hasUDSuffix())
1318  Diag(Result, diag::err_invalid_string_udl);
1319 
1320  if (AllowMacroExpansion)
1321  Lex(Result);
1322  else
1323  LexUnexpandedToken(Result);
1324  } while (Result.is(tok::string_literal));
1325 
1326  // Concatenate and parse the strings.
1327  StringLiteralParser Literal(StrToks, *this);
1328  assert(Literal.isAscii() && "Didn't allow wide strings in");
1329 
1330  if (Literal.hadError)
1331  return false;
1332 
1333  if (Literal.Pascal) {
1334  Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1335  << /*Source='in...'*/0 << DiagnosticTag;
1336  return false;
1337  }
1338 
1339  String = Literal.GetString();
1340  return true;
1341 }
1342 
1344  assert(Tok.is(tok::numeric_constant));
1345  SmallString<8> IntegerBuffer;
1346  bool NumberInvalid = false;
1347  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1348  if (NumberInvalid)
1349  return false;
1350  NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
1351  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1352  return false;
1353  llvm::APInt APVal(64, 0);
1354  if (Literal.GetIntegerValue(APVal))
1355  return false;
1356  Lex(Tok);
1357  Value = APVal.getLimitedValue();
1358  return true;
1359 }
1360 
1362  assert(Handler && "NULL comment handler");
1363  assert(llvm::find(CommentHandlers, Handler) == CommentHandlers.end() &&
1364  "Comment handler already registered");
1365  CommentHandlers.push_back(Handler);
1366 }
1367 
1369  std::vector<CommentHandler *>::iterator Pos =
1370  llvm::find(CommentHandlers, Handler);
1371  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1372  CommentHandlers.erase(Pos);
1373 }
1374 
1376  bool AnyPendingTokens = false;
1377  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1378  HEnd = CommentHandlers.end();
1379  H != HEnd; ++H) {
1380  if ((*H)->HandleComment(*this, Comment))
1381  AnyPendingTokens = true;
1382  }
1383  if (!AnyPendingTokens || getCommentRetentionState())
1384  return false;
1385  Lex(result);
1386  return true;
1387 }
1388 
1389 ModuleLoader::~ModuleLoader() = default;
1390 
1392 
1394 
1396  if (Record)
1397  return;
1398 
1399  Record = new PreprocessingRecord(getSourceManager());
1400  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1401 }
StringRef Identifier
Definition: Format.cpp:1815
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void FinalizeForModelFile()
Cleanup after model file parsing.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:259
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:97
Defines the SourceManager interface.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Defines the clang::Module class, which describes a module in the source code.
Module * getCurrentModule()
Retrieves the module that we&#39;re currently building, if any.
Defines the FileSystemStatCache interface.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:384
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:236
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:242
tok::TokenKind getKind() const
Definition: Token.h:92
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
bool LexAfterModuleImport(Token &Result)
Lex a token following the &#39;import&#39; contextual keyword.
void print(raw_ostream &OS, const SourceManager &SM) const
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:209
Definition: Format.h:2392
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void setKind(tok::TokenKind K)
Definition: Token.h:93
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive...
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:904
Describes a module or submodule.
Definition: Module.h:64
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
virtual ~ModuleLoader()
bool isInvalid() const
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:909
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:235
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
Definition: Token.h:293
const FormatToken & Tok
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:149
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
bool HandleComment(Token &result, SourceRange Comment)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
Provides lookups to, and iteration over, IdentiferInfo objects.
bool isValid() const
Exposes information about the current target.
Definition: TargetInfo.h:163
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
SourceLocation End
void makeModuleVisible(Module *M, SourceLocation Loc)
SourceLocation getLocation() const
Definition: MacroInfo.h:462
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
Definition: Token.h:289
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
Defines the clang::Preprocessor interface.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
size_t getTotalMemory() const
bool getCommentRetentionState() const
Definition: Preprocessor.h:943
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isObjectLike() const
Definition: MacroInfo.h:200
void IncrementIncludeCount(const FileEntry *File)
Increment the count for the number of times the specified FileEntry has been entered.
Definition: HeaderSearch.h:454
The result type of a method or function.
StringRef GetString() const
const SourceManager & SM
Definition: Format.cpp:1667
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:908
void DumpMacro(const MacroInfo &MI) const
void overrideFileContents(const FileEntry *SourceFile, llvm::MemoryBuffer *Buffer, bool DoNotFree)
Override the contents of the given source file by providing an already-allocated buffer.
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:921
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6...
Definition: Token.h:276
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:135
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
All of the names in this module are hidden.
Definition: Module.h:275
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:179
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:78
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:188
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:268
llvm::APInt APInt
Definition: Integral.h:27
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool getFlag(TokenFlags Flag) const
Get the specified flag.
Definition: Token.h:241
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:233
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
Optional< FileEntryRef > LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
const llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
FileID getMainFileID() const
Returns the FileID of the main source file.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:129
bool isMacroID() const
void PoisonSEHIdentifiers(bool Poison=true)
void setLiteralData(const char *Ptr)
Definition: Token.h:221
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:96
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
Module * lookupModule(StringRef ModuleName, bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
#define CXX11_KEYWORD(NAME, FLAGS)
Defines the clang::SourceLocation class and associated facilities.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:988
StringRef getRawIdentifier() const
getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode), returns a reference to the text substring in the buffer if known.
Definition: Token.h:205
SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc, SourceLocation TokenStart, SourceLocation TokenEnd)
Return a new SourceLocation that encodes that the token starting at TokenStart ends prematurely at To...
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:378
void DumpLocation(SourceLocation Loc) const
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Definition: DiagnosticIDs.h:60
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:24
Defines the clang::TargetInfo interface.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:285
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber...
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
bool isModulesImport() const
Determine whether this is the contextual keyword import.
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void setLocation(SourceLocation L)
Definition: Token.h:134
#define true
Definition: stdbool.h:16
A trivial tuple used to represent a source range.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:296
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Defines the PreprocessorLexer interface.
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:24
void InitializeTarget(const TargetInfo &Target, const TargetInfo *AuxTarget)
Perform target-specific initialization.
Definition: Builtins.cpp:42
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:272
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:996
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Definition: Token.h:171
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
SourceLocation getEndLoc() const
Definition: Token.h:153