clang  10.0.0svn
Preprocessor.cpp
Go to the documentation of this file.
1 //===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Preprocessor interface.
10 //
11 //===----------------------------------------------------------------------===//
12 //
13 // Options to support:
14 // -H - Print the name of each header file used.
15 // -d[DNI] - Dump various things.
16 // -fworking-directory - #line's with preprocessor's working dir.
17 // -fpreprocessed
18 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19 // -W*
20 // -w
21 //
22 // Messages to emit:
23 // "Multiple include guards may be useful for:\n"
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "clang/Lex/Preprocessor.h"
31 #include "clang/Basic/LLVM.h"
33 #include "clang/Basic/Module.h"
36 #include "clang/Basic/TargetInfo.h"
39 #include "clang/Lex/HeaderSearch.h"
41 #include "clang/Lex/Lexer.h"
43 #include "clang/Lex/MacroArgs.h"
44 #include "clang/Lex/MacroInfo.h"
45 #include "clang/Lex/ModuleLoader.h"
46 #include "clang/Lex/Pragma.h"
51 #include "clang/Lex/Token.h"
52 #include "clang/Lex/TokenLexer.h"
53 #include "llvm/ADT/APInt.h"
54 #include "llvm/ADT/ArrayRef.h"
55 #include "llvm/ADT/DenseMap.h"
56 #include "llvm/ADT/SmallString.h"
57 #include "llvm/ADT/SmallVector.h"
58 #include "llvm/ADT/STLExtras.h"
59 #include "llvm/ADT/StringRef.h"
60 #include "llvm/ADT/StringSwitch.h"
61 #include "llvm/Support/Capacity.h"
62 #include "llvm/Support/ErrorHandling.h"
63 #include "llvm/Support/MemoryBuffer.h"
64 #include "llvm/Support/raw_ostream.h"
65 #include <algorithm>
66 #include <cassert>
67 #include <memory>
68 #include <string>
69 #include <utility>
70 #include <vector>
71 
72 using namespace clang;
73 
74 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
75 
77 
78 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
79  DiagnosticsEngine &diags, LangOptions &opts,
80  SourceManager &SM, HeaderSearch &Headers,
81  ModuleLoader &TheModuleLoader,
82  IdentifierInfoLookup *IILookup, bool OwnsHeaders,
83  TranslationUnitKind TUKind)
84  : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
85  FileMgr(Headers.getFileMgr()), SourceMgr(SM),
86  ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
87  TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
88  // As the language options may have not been loaded yet (when
89  // deserializing an ASTUnit), adding keywords to the identifier table is
90  // deferred to Preprocessor::Initialize().
91  Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
92  TUKind(TUKind), SkipMainFilePreamble(0, true),
93  CurSubmoduleState(&NullSubmoduleState) {
94  OwnsHeaderSearch = OwnsHeaders;
95 
96  // Default to discarding comments.
97  KeepComments = false;
98  KeepMacroComments = false;
99  SuppressIncludeNotFoundError = false;
100 
101  // Macro expansion is enabled.
102  DisableMacroExpansion = false;
103  MacroExpansionInDirectivesOverride = false;
104  InMacroArgs = false;
105  ArgMacro = nullptr;
106  InMacroArgPreExpansion = false;
107  NumCachedTokenLexers = 0;
108  PragmasEnabled = true;
109  ParsingIfOrElifDirective = false;
110  PreprocessedOutput = false;
111 
112  // We haven't read anything from the external source.
113  ReadMacrosFromExternalSource = false;
114 
115  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
116  // a macro. They get unpoisoned where it is allowed.
117  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
118  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
119  if (getLangOpts().CPlusPlus2a) {
120  (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
121  SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
122  } else {
123  Ident__VA_OPT__ = nullptr;
124  }
125 
126  // Initialize the pragma handlers.
127  RegisterBuiltinPragmas();
128 
129  // Initialize builtin macros like __LINE__ and friends.
130  RegisterBuiltinMacros();
131 
132  if(LangOpts.Borland) {
133  Ident__exception_info = getIdentifierInfo("_exception_info");
134  Ident___exception_info = getIdentifierInfo("__exception_info");
135  Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
136  Ident__exception_code = getIdentifierInfo("_exception_code");
137  Ident___exception_code = getIdentifierInfo("__exception_code");
138  Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
139  Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
140  Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
141  Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
142  } else {
143  Ident__exception_info = Ident__exception_code = nullptr;
144  Ident__abnormal_termination = Ident___exception_info = nullptr;
145  Ident___exception_code = Ident___abnormal_termination = nullptr;
146  Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
147  Ident_AbnormalTermination = nullptr;
148  }
149 
150  // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
152  SkippingUntilPragmaHdrStop = true;
153 
154  // If using a PCH with a through header, start skipping tokens.
155  if (!this->PPOpts->PCHThroughHeader.empty() &&
156  !this->PPOpts->ImplicitPCHInclude.empty())
157  SkippingUntilPCHThroughHeader = true;
158 
159  if (this->PPOpts->GeneratePreamble)
160  PreambleConditionalStack.startRecording();
161 }
162 
164  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
165 
166  IncludeMacroStack.clear();
167 
168  // Destroy any macro definitions.
169  while (MacroInfoChain *I = MIChainHead) {
170  MIChainHead = I->Next;
171  I->~MacroInfoChain();
172  }
173 
174  // Free any cached macro expanders.
175  // This populates MacroArgCache, so all TokenLexers need to be destroyed
176  // before the code below that frees up the MacroArgCache list.
177  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
178  CurTokenLexer.reset();
179 
180  // Free any cached MacroArgs.
181  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
182  ArgList = ArgList->deallocate();
183 
184  // Delete the header search info, if we own it.
185  if (OwnsHeaderSearch)
186  delete &HeaderInfo;
187 }
188 
190  const TargetInfo *AuxTarget) {
191  assert((!this->Target || this->Target == &Target) &&
192  "Invalid override of target information");
193  this->Target = &Target;
194 
195  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
196  "Invalid override of aux target information.");
197  this->AuxTarget = AuxTarget;
198 
199  // Initialize information about built-ins.
200  BuiltinInfo.InitializeTarget(Target, AuxTarget);
201  HeaderInfo.setTarget(Target);
202 
203  // Populate the identifier table with info about keywords for the current language.
204  Identifiers.AddKeywords(LangOpts);
205 }
206 
208  NumEnteredSourceFiles = 0;
209 
210  // Reset pragmas
211  PragmaHandlersBackup = std::move(PragmaHandlers);
212  PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
213  RegisterBuiltinPragmas();
214 
215  // Reset PredefinesFileID
216  PredefinesFileID = FileID();
217 }
218 
220  NumEnteredSourceFiles = 1;
221 
222  PragmaHandlers = std::move(PragmaHandlersBackup);
223 }
224 
225 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
226  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
227  << getSpelling(Tok) << "'";
228 
229  if (!DumpFlags) return;
230 
231  llvm::errs() << "\t";
232  if (Tok.isAtStartOfLine())
233  llvm::errs() << " [StartOfLine]";
234  if (Tok.hasLeadingSpace())
235  llvm::errs() << " [LeadingSpace]";
236  if (Tok.isExpandDisabled())
237  llvm::errs() << " [ExpandDisabled]";
238  if (Tok.needsCleaning()) {
239  const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
240  llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
241  << "']";
242  }
243 
244  llvm::errs() << "\tLoc=<";
245  DumpLocation(Tok.getLocation());
246  llvm::errs() << ">";
247 }
248 
250  Loc.print(llvm::errs(), SourceMgr);
251 }
252 
253 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
254  llvm::errs() << "MACRO: ";
255  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
257  llvm::errs() << " ";
258  }
259  llvm::errs() << "\n";
260 }
261 
263  llvm::errs() << "\n*** Preprocessor Stats:\n";
264  llvm::errs() << NumDirectives << " directives found:\n";
265  llvm::errs() << " " << NumDefined << " #define.\n";
266  llvm::errs() << " " << NumUndefined << " #undef.\n";
267  llvm::errs() << " #include/#include_next/#import:\n";
268  llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
269  llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
270  llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
271  llvm::errs() << " " << NumElse << " #else/#elif.\n";
272  llvm::errs() << " " << NumEndif << " #endif.\n";
273  llvm::errs() << " " << NumPragma << " #pragma.\n";
274  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
275 
276  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
277  << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
278  << NumFastMacroExpanded << " on the fast path.\n";
279  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
280  << " token paste (##) operations performed, "
281  << NumFastTokenPaste << " on the fast path.\n";
282 
283  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
284 
285  llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
286  llvm::errs() << "\n Macro Expanded Tokens: "
287  << llvm::capacity_in_bytes(MacroExpandedTokens);
288  llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
289  // FIXME: List information for all submodules.
290  llvm::errs() << "\n Macros: "
291  << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
292  llvm::errs() << "\n #pragma push_macro Info: "
293  << llvm::capacity_in_bytes(PragmaPushMacroInfo);
294  llvm::errs() << "\n Poison Reasons: "
295  << llvm::capacity_in_bytes(PoisonReasons);
296  llvm::errs() << "\n Comment Handlers: "
297  << llvm::capacity_in_bytes(CommentHandlers) << "\n";
298 }
299 
301 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
302  if (IncludeExternalMacros && ExternalSource &&
303  !ReadMacrosFromExternalSource) {
304  ReadMacrosFromExternalSource = true;
305  ExternalSource->ReadDefinedMacros();
306  }
307 
308  // Make sure we cover all macros in visible modules.
309  for (const ModuleMacro &Macro : ModuleMacros)
310  CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
311 
312  return CurSubmoduleState->Macros.begin();
313 }
314 
316  return BP.getTotalMemory()
317  + llvm::capacity_in_bytes(MacroExpandedTokens)
318  + Predefines.capacity() /* Predefines buffer. */
319  // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
320  // and ModuleMacros.
321  + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
322  + llvm::capacity_in_bytes(PragmaPushMacroInfo)
323  + llvm::capacity_in_bytes(PoisonReasons)
324  + llvm::capacity_in_bytes(CommentHandlers);
325 }
326 
328 Preprocessor::macro_end(bool IncludeExternalMacros) const {
329  if (IncludeExternalMacros && ExternalSource &&
330  !ReadMacrosFromExternalSource) {
331  ReadMacrosFromExternalSource = true;
332  ExternalSource->ReadDefinedMacros();
333  }
334 
335  return CurSubmoduleState->Macros.end();
336 }
337 
338 /// Compares macro tokens with a specified token value sequence.
339 static bool MacroDefinitionEquals(const MacroInfo *MI,
340  ArrayRef<TokenValue> Tokens) {
341  return Tokens.size() == MI->getNumTokens() &&
342  std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
343 }
344 
346  SourceLocation Loc,
347  ArrayRef<TokenValue> Tokens) const {
348  SourceLocation BestLocation;
349  StringRef BestSpelling;
351  I != E; ++I) {
353  Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
354  if (!Def || !Def.getMacroInfo())
355  continue;
356  if (!Def.getMacroInfo()->isObjectLike())
357  continue;
358  if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
359  continue;
360  SourceLocation Location = Def.getLocation();
361  // Choose the macro defined latest.
362  if (BestLocation.isInvalid() ||
363  (Location.isValid() &&
364  SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
365  BestLocation = Location;
366  BestSpelling = I->first->getName();
367  }
368  }
369  return BestSpelling;
370 }
371 
373  if (CurLexer)
374  CurLexerKind = CLK_Lexer;
375  else if (CurTokenLexer)
376  CurLexerKind = CLK_TokenLexer;
377  else
378  CurLexerKind = CLK_CachingLexer;
379 }
380 
382  unsigned CompleteLine,
383  unsigned CompleteColumn) {
384  assert(File);
385  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
386  assert(!CodeCompletionFile && "Already set");
387 
388  using llvm::MemoryBuffer;
389 
390  // Load the actual file's contents.
391  bool Invalid = false;
392  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
393  if (Invalid)
394  return true;
395 
396  // Find the byte position of the truncation point.
397  const char *Position = Buffer->getBufferStart();
398  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
399  for (; *Position; ++Position) {
400  if (*Position != '\r' && *Position != '\n')
401  continue;
402 
403  // Eat \r\n or \n\r as a single line.
404  if ((Position[1] == '\r' || Position[1] == '\n') &&
405  Position[0] != Position[1])
406  ++Position;
407  ++Position;
408  break;
409  }
410  }
411 
412  Position += CompleteColumn - 1;
413 
414  // If pointing inside the preamble, adjust the position at the beginning of
415  // the file after the preamble.
416  if (SkipMainFilePreamble.first &&
417  SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
418  if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
419  Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
420  }
421 
422  if (Position > Buffer->getBufferEnd())
423  Position = Buffer->getBufferEnd();
424 
425  CodeCompletionFile = File;
426  CodeCompletionOffset = Position - Buffer->getBufferStart();
427 
428  auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
429  Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
430  char *NewBuf = NewBuffer->getBufferStart();
431  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
432  *NewPos = '\0';
433  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
434  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
435 
436  return false;
437 }
438 
439 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
440  bool IsAngled) {
441  if (CodeComplete)
442  CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
444 }
445 
447  if (CodeComplete)
448  CodeComplete->CodeCompleteNaturalLanguage();
450 }
451 
452 /// getSpelling - This method is used to get the spelling of a token into a
453 /// SmallVector. Note that the returned StringRef may not point to the
454 /// supplied buffer if a copy can be avoided.
456  SmallVectorImpl<char> &Buffer,
457  bool *Invalid) const {
458  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
459  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
460  // Try the fast path.
461  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
462  return II->getName();
463  }
464 
465  // Resize the buffer if we need to copy into it.
466  if (Tok.needsCleaning())
467  Buffer.resize(Tok.getLength());
468 
469  const char *Ptr = Buffer.data();
470  unsigned Len = getSpelling(Tok, Ptr, Invalid);
471  return StringRef(Ptr, Len);
472 }
473 
474 /// CreateString - Plop the specified string into a scratch buffer and return a
475 /// location for it. If specified, the source location provides a source
476 /// location for the token.
477 void Preprocessor::CreateString(StringRef Str, Token &Tok,
478  SourceLocation ExpansionLocStart,
479  SourceLocation ExpansionLocEnd) {
480  Tok.setLength(Str.size());
481 
482  const char *DestPtr;
483  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
484 
485  if (ExpansionLocStart.isValid())
486  Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
487  ExpansionLocEnd, Str.size());
488  Tok.setLocation(Loc);
489 
490  // If this is a raw identifier or a literal token, set the pointer data.
491  if (Tok.is(tok::raw_identifier))
492  Tok.setRawIdentifierData(DestPtr);
493  else if (Tok.isLiteral())
494  Tok.setLiteralData(DestPtr);
495 }
496 
498  auto &SM = getSourceManager();
499  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
500  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
501  bool Invalid = false;
502  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
503  if (Invalid)
504  return SourceLocation();
505 
506  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
507  const char *DestPtr;
508  SourceLocation Spelling =
509  ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
510  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
511 }
512 
514  if (!getLangOpts().isCompilingModule())
515  return nullptr;
516 
517  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
518 }
519 
520 //===----------------------------------------------------------------------===//
521 // Preprocessor Initialization Methods
522 //===----------------------------------------------------------------------===//
523 
524 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
525 /// which implicitly adds the builtin defines etc.
527  // We do not allow the preprocessor to reenter the main file. Doing so will
528  // cause FileID's to accumulate information from both runs (e.g. #line
529  // information) and predefined macros aren't guaranteed to be set properly.
530  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
531  FileID MainFileID = SourceMgr.getMainFileID();
532 
533  // If MainFileID is loaded it means we loaded an AST file, no need to enter
534  // a main file.
535  if (!SourceMgr.isLoadedFileID(MainFileID)) {
536  // Enter the main file source buffer.
537  EnterSourceFile(MainFileID, nullptr, SourceLocation());
538 
539  // If we've been asked to skip bytes in the main file (e.g., as part of a
540  // precompiled preamble), do so now.
541  if (SkipMainFilePreamble.first > 0)
542  CurLexer->SetByteOffset(SkipMainFilePreamble.first,
543  SkipMainFilePreamble.second);
544 
545  // Tell the header info that the main file was entered. If the file is later
546  // #imported, it won't be re-entered.
547  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
548  HeaderInfo.IncrementIncludeCount(FE);
549  }
550 
551  // Preprocess Predefines to populate the initial preprocessor state.
552  std::unique_ptr<llvm::MemoryBuffer> SB =
553  llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
554  assert(SB && "Cannot create predefined source buffer");
555  FileID FID = SourceMgr.createFileID(std::move(SB));
556  assert(FID.isValid() && "Could not create FileID for predefines?");
557  setPredefinesFileID(FID);
558 
559  // Start parsing the predefines.
560  EnterSourceFile(FID, nullptr, SourceLocation());
561 
562  if (!PPOpts->PCHThroughHeader.empty()) {
563  // Lookup and save the FileID for the through header. If it isn't found
564  // in the search path, it's a fatal error.
565  const DirectoryLookup *CurDir;
566  const FileEntry *File = LookupFile(
567  SourceLocation(), PPOpts->PCHThroughHeader,
568  /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, CurDir,
569  /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
570  /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
571  /*IsFrameworkFound=*/nullptr);
572  if (!File) {
573  Diag(SourceLocation(), diag::err_pp_through_header_not_found)
574  << PPOpts->PCHThroughHeader;
575  return;
576  }
577  setPCHThroughHeaderFileID(
578  SourceMgr.createFileID(File, SourceLocation(), SrcMgr::C_User));
579  }
580 
581  // Skip tokens from the Predefines and if needed the main file.
582  if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
583  (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
585 }
586 
587 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
588  assert(PCHThroughHeaderFileID.isInvalid() &&
589  "PCHThroughHeaderFileID already set!");
590  PCHThroughHeaderFileID = FID;
591 }
592 
594  assert(PCHThroughHeaderFileID.isValid() &&
595  "Invalid PCH through header FileID");
596  return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
597 }
598 
600  return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
601  PCHThroughHeaderFileID.isValid();
602 }
603 
605  return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
606  PCHThroughHeaderFileID.isValid();
607 }
608 
610  return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
611 }
612 
614  return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
615 }
616 
617 /// Skip tokens until after the #include of the through header or
618 /// until after a #pragma hdrstop is seen. Tokens in the predefines file
619 /// and the main file may be skipped. If the end of the predefines file
620 /// is reached, skipping continues into the main file. If the end of the
621 /// main file is reached, it's a fatal error.
623  bool ReachedMainFileEOF = false;
624  bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
625  bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
626  Token Tok;
627  while (true) {
628  bool InPredefines =
629  (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
630  switch (CurLexerKind) {
631  case CLK_Lexer:
632  CurLexer->Lex(Tok);
633  break;
634  case CLK_TokenLexer:
635  CurTokenLexer->Lex(Tok);
636  break;
637  case CLK_CachingLexer:
638  CachingLex(Tok);
639  break;
640  case CLK_LexAfterModuleImport:
642  break;
643  }
644  if (Tok.is(tok::eof) && !InPredefines) {
645  ReachedMainFileEOF = true;
646  break;
647  }
648  if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
649  break;
650  if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
651  break;
652  }
653  if (ReachedMainFileEOF) {
654  if (UsingPCHThroughHeader)
655  Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
656  << PPOpts->PCHThroughHeader << 1;
657  else if (!PPOpts->PCHWithHdrStopCreate)
658  Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
659  }
660 }
661 
662 void Preprocessor::replayPreambleConditionalStack() {
663  // Restore the conditional stack from the preamble, if there is one.
664  if (PreambleConditionalStack.isReplaying()) {
665  assert(CurPPLexer &&
666  "CurPPLexer is null when calling replayPreambleConditionalStack.");
667  CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
668  PreambleConditionalStack.doneReplaying();
669  if (PreambleConditionalStack.reachedEOFWhileSkipping())
670  SkipExcludedConditionalBlock(
671  PreambleConditionalStack.SkipInfo->HashTokenLoc,
672  PreambleConditionalStack.SkipInfo->IfTokenLoc,
673  PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
674  PreambleConditionalStack.SkipInfo->FoundElse,
675  PreambleConditionalStack.SkipInfo->ElseLoc);
676  }
677 }
678 
680  // Notify the client that we reached the end of the source file.
681  if (Callbacks)
682  Callbacks->EndOfMainFile();
683 }
684 
685 //===----------------------------------------------------------------------===//
686 // Lexer Event Handling.
687 //===----------------------------------------------------------------------===//
688 
689 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
690 /// identifier information for the token and install it into the token,
691 /// updating the token kind accordingly.
693  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
694 
695  // Look up this token, see if it is a macro, or if it is a language keyword.
696  IdentifierInfo *II;
697  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
698  // No cleaning needed, just use the characters from the lexed buffer.
699  II = getIdentifierInfo(Identifier.getRawIdentifier());
700  } else {
701  // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
702  SmallString<64> IdentifierBuffer;
703  StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
704 
705  if (Identifier.hasUCN()) {
706  SmallString<64> UCNIdentifierBuffer;
707  expandUCNs(UCNIdentifierBuffer, CleanedStr);
708  II = getIdentifierInfo(UCNIdentifierBuffer);
709  } else {
710  II = getIdentifierInfo(CleanedStr);
711  }
712  }
713 
714  // Update the token info (identifier info and appropriate token kind).
715  Identifier.setIdentifierInfo(II);
716  if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
718  Identifier.setKind(tok::identifier);
719  else
720  Identifier.setKind(II->getTokenID());
721 
722  return II;
723 }
724 
725 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
726  PoisonReasons[II] = DiagID;
727 }
728 
730  assert(Ident__exception_code && Ident__exception_info);
731  assert(Ident___exception_code && Ident___exception_info);
732  Ident__exception_code->setIsPoisoned(Poison);
733  Ident___exception_code->setIsPoisoned(Poison);
734  Ident_GetExceptionCode->setIsPoisoned(Poison);
735  Ident__exception_info->setIsPoisoned(Poison);
736  Ident___exception_info->setIsPoisoned(Poison);
737  Ident_GetExceptionInfo->setIsPoisoned(Poison);
738  Ident__abnormal_termination->setIsPoisoned(Poison);
739  Ident___abnormal_termination->setIsPoisoned(Poison);
740  Ident_AbnormalTermination->setIsPoisoned(Poison);
741 }
742 
744  assert(Identifier.getIdentifierInfo() &&
745  "Can't handle identifiers without identifier info!");
746  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
747  PoisonReasons.find(Identifier.getIdentifierInfo());
748  if(it == PoisonReasons.end())
749  Diag(Identifier, diag::err_pp_used_poisoned_id);
750  else
751  Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
752 }
753 
754 /// Returns a diagnostic message kind for reporting a future keyword as
755 /// appropriate for the identifier and specified language.
757  const LangOptions &LangOpts) {
758  assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
759 
760  if (LangOpts.CPlusPlus)
761  return llvm::StringSwitch<diag::kind>(II.getName())
762 #define CXX11_KEYWORD(NAME, FLAGS) \
763  .Case(#NAME, diag::warn_cxx11_keyword)
764 #define CXX2A_KEYWORD(NAME, FLAGS) \
765  .Case(#NAME, diag::warn_cxx2a_keyword)
766 #include "clang/Basic/TokenKinds.def"
767  ;
768 
769  llvm_unreachable(
770  "Keyword not known to come from a newer Standard or proposed Standard");
771 }
772 
773 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
774  assert(II.isOutOfDate() && "not out of date");
776 }
777 
778 /// HandleIdentifier - This callback is invoked when the lexer reads an
779 /// identifier. This callback looks up the identifier in the map and/or
780 /// potentially macro expands it or turns it into a named token (like 'for').
781 ///
782 /// Note that callers of this method are guarded by checking the
783 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
784 /// IdentifierInfo methods that compute these properties will need to change to
785 /// match.
787  assert(Identifier.getIdentifierInfo() &&
788  "Can't handle identifiers without identifier info!");
789 
790  IdentifierInfo &II = *Identifier.getIdentifierInfo();
791 
792  // If the information about this identifier is out of date, update it from
793  // the external source.
794  // We have to treat __VA_ARGS__ in a special way, since it gets
795  // serialized with isPoisoned = true, but our preprocessor may have
796  // unpoisoned it if we're defining a C99 macro.
797  if (II.isOutOfDate()) {
798  bool CurrentIsPoisoned = false;
799  const bool IsSpecialVariadicMacro =
800  &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
801  if (IsSpecialVariadicMacro)
802  CurrentIsPoisoned = II.isPoisoned();
803 
804  updateOutOfDateIdentifier(II);
805  Identifier.setKind(II.getTokenID());
806 
807  if (IsSpecialVariadicMacro)
808  II.setIsPoisoned(CurrentIsPoisoned);
809  }
810 
811  // If this identifier was poisoned, and if it was not produced from a macro
812  // expansion, emit an error.
813  if (II.isPoisoned() && CurPPLexer) {
814  HandlePoisonedIdentifier(Identifier);
815  }
816 
817  // If this is a macro to be expanded, do it.
818  if (MacroDefinition MD = getMacroDefinition(&II)) {
819  auto *MI = MD.getMacroInfo();
820  assert(MI && "macro definition with no macro info?");
821  if (!DisableMacroExpansion) {
822  if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
823  // C99 6.10.3p10: If the preprocessing token immediately after the
824  // macro name isn't a '(', this macro should not be expanded.
825  if (!MI->isFunctionLike() || isNextPPTokenLParen())
826  return HandleMacroExpandedIdentifier(Identifier, MD);
827  } else {
828  // C99 6.10.3.4p2 says that a disabled macro may never again be
829  // expanded, even if it's in a context where it could be expanded in the
830  // future.
831  Identifier.setFlag(Token::DisableExpand);
832  if (MI->isObjectLike() || isNextPPTokenLParen())
833  Diag(Identifier, diag::pp_disabled_macro_expansion);
834  }
835  }
836  }
837 
838  // If this identifier is a keyword in a newer Standard or proposed Standard,
839  // produce a warning. Don't warn if we're not considering macro expansion,
840  // since this identifier might be the name of a macro.
841  // FIXME: This warning is disabled in cases where it shouldn't be, like
842  // "#define constexpr constexpr", "int constexpr;"
843  if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
844  Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
845  << II.getName();
846  // Don't diagnose this keyword again in this translation unit.
847  II.setIsFutureCompatKeyword(false);
848  }
849 
850  // If this is an extension token, diagnose its use.
851  // We avoid diagnosing tokens that originate from macro definitions.
852  // FIXME: This warning is disabled in cases where it shouldn't be,
853  // like "#define TY typeof", "TY(1) x".
854  if (II.isExtensionToken() && !DisableMacroExpansion)
855  Diag(Identifier, diag::ext_token_used);
856 
857  // If this is the 'import' contextual keyword following an '@', note
858  // that the next token indicates a module name.
859  //
860  // Note that we do not treat 'import' as a contextual
861  // keyword when we're in a caching lexer, because caching lexers only get
862  // used in contexts where import declarations are disallowed.
863  //
864  // Likewise if this is the C++ Modules TS import keyword.
865  if (((LastTokenWasAt && II.isModulesImport()) ||
866  Identifier.is(tok::kw_import)) &&
867  !InMacroArgs && !DisableMacroExpansion &&
868  (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
869  CurLexerKind != CLK_CachingLexer) {
870  ModuleImportLoc = Identifier.getLocation();
871  ModuleImportPath.clear();
872  ModuleImportExpectsIdentifier = true;
873  CurLexerKind = CLK_LexAfterModuleImport;
874  }
875  return true;
876 }
877 
879  ++LexLevel;
880 
881  // We loop here until a lex function returns a token; this avoids recursion.
882  bool ReturnedToken;
883  do {
884  switch (CurLexerKind) {
885  case CLK_Lexer:
886  ReturnedToken = CurLexer->Lex(Result);
887  break;
888  case CLK_TokenLexer:
889  ReturnedToken = CurTokenLexer->Lex(Result);
890  break;
891  case CLK_CachingLexer:
892  CachingLex(Result);
893  ReturnedToken = true;
894  break;
895  case CLK_LexAfterModuleImport:
896  ReturnedToken = LexAfterModuleImport(Result);
897  break;
898  }
899  } while (!ReturnedToken);
900 
901  if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
902  // Remember the identifier before code completion token.
905  // Set IdenfitierInfo to null to avoid confusing code that handles both
906  // identifiers and completion tokens.
907  Result.setIdentifierInfo(nullptr);
908  }
909 
910  // Update ImportSeqState to track our position within a C++20 import-seq
911  // if this token is being produced as a result of phase 4 of translation.
912  if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
913  !Result.getFlag(Token::IsReinjected)) {
914  switch (Result.getKind()) {
915  case tok::l_paren: case tok::l_square: case tok::l_brace:
916  ImportSeqState.handleOpenBracket();
917  break;
918  case tok::r_paren: case tok::r_square:
919  ImportSeqState.handleCloseBracket();
920  break;
921  case tok::r_brace:
922  ImportSeqState.handleCloseBrace();
923  break;
924  case tok::semi:
925  ImportSeqState.handleSemi();
926  break;
927  case tok::header_name:
928  case tok::annot_header_unit:
929  ImportSeqState.handleHeaderName();
930  break;
931  case tok::kw_export:
932  ImportSeqState.handleExport();
933  break;
934  case tok::identifier:
935  if (Result.getIdentifierInfo()->isModulesImport()) {
936  ImportSeqState.handleImport();
937  if (ImportSeqState.afterImportSeq()) {
938  ModuleImportLoc = Result.getLocation();
939  ModuleImportPath.clear();
940  ModuleImportExpectsIdentifier = true;
941  CurLexerKind = CLK_LexAfterModuleImport;
942  }
943  break;
944  }
945  LLVM_FALLTHROUGH;
946  default:
947  ImportSeqState.handleMisc();
948  break;
949  }
950  }
951 
952  LastTokenWasAt = Result.is(tok::at);
953  --LexLevel;
954  if (OnToken && LexLevel == 0 && !Result.getFlag(Token::IsReinjected))
955  OnToken(Result);
956 }
957 
958 /// Lex a header-name token (including one formed from header-name-tokens if
959 /// \p AllowConcatenation is \c true).
960 ///
961 /// \param FilenameTok Filled in with the next token. On success, this will
962 /// be either a header_name token. On failure, it will be whatever other
963 /// token was found instead.
964 /// \param AllowMacroExpansion If \c true, allow the header name to be formed
965 /// by macro expansion (concatenating tokens as necessary if the first
966 /// token is a '<').
967 /// \return \c true if we reached EOD or EOF while looking for a > token in
968 /// a concatenated header name and diagnosed it. \c false otherwise.
969 bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
970  // Lex using header-name tokenization rules if tokens are being lexed from
971  // a file. Just grab a token normally if we're in a macro expansion.
972  if (CurPPLexer)
973  CurPPLexer->LexIncludeFilename(FilenameTok);
974  else
975  Lex(FilenameTok);
976 
977  // This could be a <foo/bar.h> file coming from a macro expansion. In this
978  // case, glue the tokens together into an angle_string_literal token.
979  SmallString<128> FilenameBuffer;
980  if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
981  bool StartOfLine = FilenameTok.isAtStartOfLine();
982  bool LeadingSpace = FilenameTok.hasLeadingSpace();
983  bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
984 
985  SourceLocation Start = FilenameTok.getLocation();
987  FilenameBuffer.push_back('<');
988 
989  // Consume tokens until we find a '>'.
990  // FIXME: A header-name could be formed starting or ending with an
991  // alternative token. It's not clear whether that's ill-formed in all
992  // cases.
993  while (FilenameTok.isNot(tok::greater)) {
994  Lex(FilenameTok);
995  if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
996  Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
997  Diag(Start, diag::note_matching) << tok::less;
998  return true;
999  }
1000 
1001  End = FilenameTok.getLocation();
1002 
1003  // FIXME: Provide code completion for #includes.
1004  if (FilenameTok.is(tok::code_completion)) {
1006  Lex(FilenameTok);
1007  continue;
1008  }
1009 
1010  // Append the spelling of this token to the buffer. If there was a space
1011  // before it, add it now.
1012  if (FilenameTok.hasLeadingSpace())
1013  FilenameBuffer.push_back(' ');
1014 
1015  // Get the spelling of the token, directly into FilenameBuffer if
1016  // possible.
1017  size_t PreAppendSize = FilenameBuffer.size();
1018  FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
1019 
1020  const char *BufPtr = &FilenameBuffer[PreAppendSize];
1021  unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
1022 
1023  // If the token was spelled somewhere else, copy it into FilenameBuffer.
1024  if (BufPtr != &FilenameBuffer[PreAppendSize])
1025  memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1026 
1027  // Resize FilenameBuffer to the correct size.
1028  if (FilenameTok.getLength() != ActualLen)
1029  FilenameBuffer.resize(PreAppendSize + ActualLen);
1030  }
1031 
1032  FilenameTok.startToken();
1033  FilenameTok.setKind(tok::header_name);
1034  FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine);
1035  FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace);
1036  FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro);
1037  CreateString(FilenameBuffer, FilenameTok, Start, End);
1038  } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
1039  // Convert a string-literal token of the form " h-char-sequence "
1040  // (produced by macro expansion) into a header-name token.
1041  //
1042  // The rules for header-names don't quite match the rules for
1043  // string-literals, but all the places where they differ result in
1044  // undefined behavior, so we can and do treat them the same.
1045  //
1046  // A string-literal with a prefix or suffix is not translated into a
1047  // header-name. This could theoretically be observable via the C++20
1048  // context-sensitive header-name formation rules.
1049  StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
1050  if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1051  FilenameTok.setKind(tok::header_name);
1052  }
1053 
1054  return false;
1055 }
1056 
1057 /// Collect the tokens of a C++20 pp-import-suffix.
1059  // FIXME: For error recovery, consider recognizing attribute syntax here
1060  // and terminating / diagnosing a missing semicolon if we find anything
1061  // else? (Can we leave that to the parser?)
1062  unsigned BracketDepth = 0;
1063  while (true) {
1064  Toks.emplace_back();
1065  Lex(Toks.back());
1066 
1067  switch (Toks.back().getKind()) {
1068  case tok::l_paren: case tok::l_square: case tok::l_brace:
1069  ++BracketDepth;
1070  break;
1071 
1072  case tok::r_paren: case tok::r_square: case tok::r_brace:
1073  if (BracketDepth == 0)
1074  return;
1075  --BracketDepth;
1076  break;
1077 
1078  case tok::semi:
1079  if (BracketDepth == 0)
1080  return;
1081  break;
1082 
1083  case tok::eof:
1084  return;
1085 
1086  default:
1087  break;
1088  }
1089  }
1090 }
1091 
1092 
1093 /// Lex a token following the 'import' contextual keyword.
1094 ///
1095 /// pp-import: [C++20]
1096 /// import header-name pp-import-suffix[opt] ;
1097 /// import header-name-tokens pp-import-suffix[opt] ;
1098 /// [ObjC] @ import module-name ;
1099 /// [Clang] import module-name ;
1100 ///
1101 /// header-name-tokens:
1102 /// string-literal
1103 /// < [any sequence of preprocessing-tokens other than >] >
1104 ///
1105 /// module-name:
1106 /// module-name-qualifier[opt] identifier
1107 ///
1108 /// module-name-qualifier
1109 /// module-name-qualifier[opt] identifier .
1110 ///
1111 /// We respond to a pp-import by importing macros from the named module.
1113  // Figure out what kind of lexer we actually have.
1115 
1116  // Lex the next token. The header-name lexing rules are used at the start of
1117  // a pp-import.
1118  //
1119  // For now, we only support header-name imports in C++20 mode.
1120  // FIXME: Should we allow this in all language modes that support an import
1121  // declaration as an extension?
1122  if (ModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1123  if (LexHeaderName(Result))
1124  return true;
1125  } else {
1126  Lex(Result);
1127  }
1128 
1129  // Allocate a holding buffer for a sequence of tokens and introduce it into
1130  // the token stream.
1131  auto EnterTokens = [this](ArrayRef<Token> Toks) {
1132  auto ToksCopy = llvm::make_unique<Token[]>(Toks.size());
1133  std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1134  EnterTokenStream(std::move(ToksCopy), Toks.size(),
1135  /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1136  };
1137 
1138  // Check for a header-name.
1139  SmallVector<Token, 32> Suffix;
1140  if (Result.is(tok::header_name)) {
1141  // Enter the header-name token into the token stream; a Lex action cannot
1142  // both return a token and cache tokens (doing so would corrupt the token
1143  // cache if the call to Lex comes from CachingLex / PeekAhead).
1144  Suffix.push_back(Result);
1145 
1146  // Consume the pp-import-suffix and expand any macros in it now. We'll add
1147  // it back into the token stream later.
1148  CollectPpImportSuffix(Suffix);
1149  if (Suffix.back().isNot(tok::semi)) {
1150  // This is not a pp-import after all.
1151  EnterTokens(Suffix);
1152  return false;
1153  }
1154 
1155  // C++2a [cpp.module]p1:
1156  // The ';' preprocessing-token terminating a pp-import shall not have
1157  // been produced by macro replacement.
1158  SourceLocation SemiLoc = Suffix.back().getLocation();
1159  if (SemiLoc.isMacroID())
1160  Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1161 
1162  // Reconstitute the import token.
1163  Token ImportTok;
1164  ImportTok.startToken();
1165  ImportTok.setKind(tok::kw_import);
1166  ImportTok.setLocation(ModuleImportLoc);
1167  ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
1168  ImportTok.setLength(6);
1169 
1170  auto Action = HandleHeaderIncludeOrImport(
1171  /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
1172  switch (Action.Kind) {
1173  case ImportAction::None:
1174  break;
1175 
1176  case ImportAction::ModuleBegin:
1177  // Let the parser know we're textually entering the module.
1178  Suffix.emplace_back();
1179  Suffix.back().startToken();
1180  Suffix.back().setKind(tok::annot_module_begin);
1181  Suffix.back().setLocation(SemiLoc);
1182  Suffix.back().setAnnotationEndLoc(SemiLoc);
1183  Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1184  LLVM_FALLTHROUGH;
1185 
1186  case ImportAction::ModuleImport:
1187  case ImportAction::SkippedModuleImport:
1188  // We chose to import (or textually enter) the file. Convert the
1189  // header-name token into a header unit annotation token.
1190  Suffix[0].setKind(tok::annot_header_unit);
1191  Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1192  Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1193  // FIXME: Call the moduleImport callback?
1194  break;
1195  }
1196 
1197  EnterTokens(Suffix);
1198  return false;
1199  }
1200 
1201  // The token sequence
1202  //
1203  // import identifier (. identifier)*
1204  //
1205  // indicates a module import directive. We already saw the 'import'
1206  // contextual keyword, so now we're looking for the identifiers.
1207  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1208  // We expected to see an identifier here, and we did; continue handling
1209  // identifiers.
1210  ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
1211  Result.getLocation()));
1212  ModuleImportExpectsIdentifier = false;
1213  CurLexerKind = CLK_LexAfterModuleImport;
1214  return true;
1215  }
1216 
1217  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1218  // see the next identifier. (We can also see a '[[' that begins an
1219  // attribute-specifier-seq here under the C++ Modules TS.)
1220  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1221  ModuleImportExpectsIdentifier = true;
1222  CurLexerKind = CLK_LexAfterModuleImport;
1223  return true;
1224  }
1225 
1226  // If we didn't recognize a module name at all, this is not a (valid) import.
1227  if (ModuleImportPath.empty() || Result.is(tok::eof))
1228  return true;
1229 
1230  // Consume the pp-import-suffix and expand any macros in it now, if we're not
1231  // at the semicolon already.
1232  SourceLocation SemiLoc = Result.getLocation();
1233  if (Result.isNot(tok::semi)) {
1234  Suffix.push_back(Result);
1235  CollectPpImportSuffix(Suffix);
1236  if (Suffix.back().isNot(tok::semi)) {
1237  // This is not an import after all.
1238  EnterTokens(Suffix);
1239  return false;
1240  }
1241  SemiLoc = Suffix.back().getLocation();
1242  }
1243 
1244  // Under the Modules TS, the dot is just part of the module name, and not
1245  // a real hierarchy separator. Flatten such module names now.
1246  //
1247  // FIXME: Is this the right level to be performing this transformation?
1248  std::string FlatModuleName;
1249  if (getLangOpts().ModulesTS || getLangOpts().CPlusPlusModules) {
1250  for (auto &Piece : ModuleImportPath) {
1251  if (!FlatModuleName.empty())
1252  FlatModuleName += ".";
1253  FlatModuleName += Piece.first->getName();
1254  }
1255  SourceLocation FirstPathLoc = ModuleImportPath[0].second;
1256  ModuleImportPath.clear();
1257  ModuleImportPath.push_back(
1258  std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1259  }
1260 
1261  Module *Imported = nullptr;
1262  if (getLangOpts().Modules) {
1263  Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1264  ModuleImportPath,
1266  /*IsInclusionDirective=*/false);
1267  if (Imported)
1268  makeModuleVisible(Imported, SemiLoc);
1269  }
1270  if (Callbacks)
1271  Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1272 
1273  if (!Suffix.empty()) {
1274  EnterTokens(Suffix);
1275  return false;
1276  }
1277  return true;
1278 }
1279 
1281  CurSubmoduleState->VisibleModules.setVisible(
1282  M, Loc, [](Module *) {},
1283  [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1284  // FIXME: Include the path in the diagnostic.
1285  // FIXME: Include the import location for the conflicting module.
1286  Diag(ModuleImportLoc, diag::warn_module_conflict)
1287  << Path[0]->getFullModuleName()
1288  << Conflict->getFullModuleName()
1289  << Message;
1290  });
1291 
1292  // Add this module to the imports list of the currently-built submodule.
1293  if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1294  BuildingSubmoduleStack.back().M->Imports.insert(M);
1295 }
1296 
1298  const char *DiagnosticTag,
1299  bool AllowMacroExpansion) {
1300  // We need at least one string literal.
1301  if (Result.isNot(tok::string_literal)) {
1302  Diag(Result, diag::err_expected_string_literal)
1303  << /*Source='in...'*/0 << DiagnosticTag;
1304  return false;
1305  }
1306 
1307  // Lex string literal tokens, optionally with macro expansion.
1308  SmallVector<Token, 4> StrToks;
1309  do {
1310  StrToks.push_back(Result);
1311 
1312  if (Result.hasUDSuffix())
1313  Diag(Result, diag::err_invalid_string_udl);
1314 
1315  if (AllowMacroExpansion)
1316  Lex(Result);
1317  else
1318  LexUnexpandedToken(Result);
1319  } while (Result.is(tok::string_literal));
1320 
1321  // Concatenate and parse the strings.
1322  StringLiteralParser Literal(StrToks, *this);
1323  assert(Literal.isAscii() && "Didn't allow wide strings in");
1324 
1325  if (Literal.hadError)
1326  return false;
1327 
1328  if (Literal.Pascal) {
1329  Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1330  << /*Source='in...'*/0 << DiagnosticTag;
1331  return false;
1332  }
1333 
1334  String = Literal.GetString();
1335  return true;
1336 }
1337 
1339  assert(Tok.is(tok::numeric_constant));
1340  SmallString<8> IntegerBuffer;
1341  bool NumberInvalid = false;
1342  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1343  if (NumberInvalid)
1344  return false;
1345  NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
1346  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1347  return false;
1348  llvm::APInt APVal(64, 0);
1349  if (Literal.GetIntegerValue(APVal))
1350  return false;
1351  Lex(Tok);
1352  Value = APVal.getLimitedValue();
1353  return true;
1354 }
1355 
1357  assert(Handler && "NULL comment handler");
1358  assert(llvm::find(CommentHandlers, Handler) == CommentHandlers.end() &&
1359  "Comment handler already registered");
1360  CommentHandlers.push_back(Handler);
1361 }
1362 
1364  std::vector<CommentHandler *>::iterator Pos =
1365  llvm::find(CommentHandlers, Handler);
1366  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1367  CommentHandlers.erase(Pos);
1368 }
1369 
1371  bool AnyPendingTokens = false;
1372  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1373  HEnd = CommentHandlers.end();
1374  H != HEnd; ++H) {
1375  if ((*H)->HandleComment(*this, Comment))
1376  AnyPendingTokens = true;
1377  }
1378  if (!AnyPendingTokens || getCommentRetentionState())
1379  return false;
1380  Lex(result);
1381  return true;
1382 }
1383 
1384 ModuleLoader::~ModuleLoader() = default;
1385 
1387 
1389 
1391  if (Record)
1392  return;
1393 
1394  Record = new PreprocessingRecord(getSourceManager());
1395  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1396 }
StringRef Identifier
Definition: Format.cpp:1719
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void FinalizeForModelFile()
Cleanup after model file parsing.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:259
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:97
Defines the SourceManager interface.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Defines the clang::Module class, which describes a module in the source code.
Module * getCurrentModule()
Retrieves the module that we&#39;re currently building, if any.
Defines the FileSystemStatCache interface.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:368
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:236
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:242
long i
Definition: xmmintrin.h:1456
tok::TokenKind getKind() const
Definition: Token.h:92
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
bool LexAfterModuleImport(Token &Result)
Lex a token following the &#39;import&#39; contextual keyword.
void print(raw_ostream &OS, const SourceManager &SM) const
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:209
Definition: Format.h:2274
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void setKind(tok::TokenKind K)
Definition: Token.h:93
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive...
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:903
Describes a module or submodule.
Definition: Module.h:64
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
virtual ~ModuleLoader()
bool isInvalid() const
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:908
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:235
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
Definition: Token.h:293
const FormatToken & Tok
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:149
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
bool HandleComment(Token &result, SourceRange Comment)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
Provides lookups to, and iteration over, IdentiferInfo objects.
bool isValid() const
Exposes information about the current target.
Definition: TargetInfo.h:161
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
SourceLocation End
void makeModuleVisible(Module *M, SourceLocation Loc)
SourceLocation getLocation() const
Definition: MacroInfo.h:462
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
Definition: Token.h:289
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
Defines the clang::Preprocessor interface.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
size_t getTotalMemory() const
bool getCommentRetentionState() const
Definition: Preprocessor.h:942
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isObjectLike() const
Definition: MacroInfo.h:200
void IncrementIncludeCount(const FileEntry *File)
Increment the count for the number of times the specified FileEntry has been entered.
Definition: HeaderSearch.h:460
The result type of a method or function.
StringRef GetString() const
const SourceManager & SM
Definition: Format.cpp:1572
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:907
void DumpMacro(const MacroInfo &MI) const
void overrideFileContents(const FileEntry *SourceFile, llvm::MemoryBuffer *Buffer, bool DoNotFree)
Override the contents of the given source file by providing an already-allocated buffer.
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:920
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6...
Definition: Token.h:276
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:135
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
All of the names in this module are hidden.
Definition: Module.h:275
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:179
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:188
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:268
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:98
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool getFlag(TokenFlags Flag) const
Get the specified flag.
Definition: Token.h:241
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:233
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
const llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
FileID getMainFileID() const
Returns the FileID of the main source file.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:99
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:129
bool isMacroID() const
void PoisonSEHIdentifiers(bool Poison=true)
void setLiteralData(const char *Ptr)
Definition: Token.h:221
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:96
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
Module * lookupModule(StringRef ModuleName, bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
#define CXX11_KEYWORD(NAME, FLAGS)
Defines the clang::SourceLocation class and associated facilities.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:987
StringRef getRawIdentifier() const
getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode), returns a reference to the text substring in the buffer if known.
Definition: Token.h:205
SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc, SourceLocation TokenStart, SourceLocation TokenEnd)
Return a new SourceLocation that encodes that the token starting at TokenStart ends prematurely at To...
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:362
void DumpLocation(SourceLocation Loc) const
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Definition: DiagnosticIDs.h:60
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:24
Defines the clang::TargetInfo interface.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:285
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber...
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
bool isModulesImport() const
Determine whether this is the contextual keyword import.
result[0]
Definition: emmintrin.h:120
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void setLocation(SourceLocation L)
Definition: Token.h:134
#define true
Definition: stdbool.h:16
A trivial tuple used to represent a source range.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:296
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Defines the PreprocessorLexer interface.
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:24
void InitializeTarget(const TargetInfo &Target, const TargetInfo *AuxTarget)
Perform target-specific initialization.
Definition: Builtins.cpp:42
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:272
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:995
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Definition: Token.h:171
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
SourceLocation getEndLoc() const
Definition: Token.h:153