clang  9.0.0svn
Preprocessor.cpp
Go to the documentation of this file.
1 //===- Preprocess.cpp - C Language Family Preprocessor Implementation -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Preprocessor interface.
10 //
11 //===----------------------------------------------------------------------===//
12 //
13 // Options to support:
14 // -H - Print the name of each header file used.
15 // -d[DNI] - Dump various things.
16 // -fworking-directory - #line's with preprocessor's working dir.
17 // -fpreprocessed
18 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19 // -W*
20 // -w
21 //
22 // Messages to emit:
23 // "Multiple include guards may be useful for:\n"
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "clang/Lex/Preprocessor.h"
31 #include "clang/Basic/LLVM.h"
33 #include "clang/Basic/Module.h"
36 #include "clang/Basic/TargetInfo.h"
39 #include "clang/Lex/HeaderSearch.h"
41 #include "clang/Lex/Lexer.h"
43 #include "clang/Lex/MacroArgs.h"
44 #include "clang/Lex/MacroInfo.h"
45 #include "clang/Lex/ModuleLoader.h"
46 #include "clang/Lex/Pragma.h"
51 #include "clang/Lex/Token.h"
52 #include "clang/Lex/TokenLexer.h"
53 #include "llvm/ADT/APInt.h"
54 #include "llvm/ADT/ArrayRef.h"
55 #include "llvm/ADT/DenseMap.h"
56 #include "llvm/ADT/SmallString.h"
57 #include "llvm/ADT/SmallVector.h"
58 #include "llvm/ADT/STLExtras.h"
59 #include "llvm/ADT/StringRef.h"
60 #include "llvm/ADT/StringSwitch.h"
61 #include "llvm/Support/Capacity.h"
62 #include "llvm/Support/ErrorHandling.h"
63 #include "llvm/Support/MemoryBuffer.h"
64 #include "llvm/Support/raw_ostream.h"
65 #include <algorithm>
66 #include <cassert>
67 #include <memory>
68 #include <string>
69 #include <utility>
70 #include <vector>
71 
72 using namespace clang;
73 
74 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
75 
77 
78 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
79  DiagnosticsEngine &diags, LangOptions &opts,
81  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
82  IdentifierInfoLookup *IILookup, bool OwnsHeaders,
83  TranslationUnitKind TUKind)
84  : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
85  FileMgr(Headers.getFileMgr()), SourceMgr(SM), PCMCache(PCMCache),
86  ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
87  TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
88  // As the language options may have not been loaded yet (when
89  // deserializing an ASTUnit), adding keywords to the identifier table is
90  // deferred to Preprocessor::Initialize().
91  Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
92  TUKind(TUKind), SkipMainFilePreamble(0, true),
93  CurSubmoduleState(&NullSubmoduleState) {
94  OwnsHeaderSearch = OwnsHeaders;
95 
96  // Default to discarding comments.
97  KeepComments = false;
98  KeepMacroComments = false;
99  SuppressIncludeNotFoundError = false;
100 
101  // Macro expansion is enabled.
102  DisableMacroExpansion = false;
103  MacroExpansionInDirectivesOverride = false;
104  InMacroArgs = false;
105  InMacroArgPreExpansion = false;
106  NumCachedTokenLexers = 0;
107  PragmasEnabled = true;
108  ParsingIfOrElifDirective = false;
109  PreprocessedOutput = false;
110 
111  // We haven't read anything from the external source.
112  ReadMacrosFromExternalSource = false;
113 
114  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
115  // a macro. They get unpoisoned where it is allowed.
116  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
117  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
118  if (getLangOpts().CPlusPlus2a) {
119  (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
120  SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
121  } else {
122  Ident__VA_OPT__ = nullptr;
123  }
124 
125  // Initialize the pragma handlers.
126  RegisterBuiltinPragmas();
127 
128  // Initialize builtin macros like __LINE__ and friends.
129  RegisterBuiltinMacros();
130 
131  if(LangOpts.Borland) {
132  Ident__exception_info = getIdentifierInfo("_exception_info");
133  Ident___exception_info = getIdentifierInfo("__exception_info");
134  Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
135  Ident__exception_code = getIdentifierInfo("_exception_code");
136  Ident___exception_code = getIdentifierInfo("__exception_code");
137  Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
138  Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
139  Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
140  Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
141  } else {
142  Ident__exception_info = Ident__exception_code = nullptr;
143  Ident__abnormal_termination = Ident___exception_info = nullptr;
144  Ident___exception_code = Ident___abnormal_termination = nullptr;
145  Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
146  Ident_AbnormalTermination = nullptr;
147  }
148 
149  // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
151  SkippingUntilPragmaHdrStop = true;
152 
153  // If using a PCH with a through header, start skipping tokens.
154  if (!this->PPOpts->PCHThroughHeader.empty() &&
155  !this->PPOpts->ImplicitPCHInclude.empty())
156  SkippingUntilPCHThroughHeader = true;
157 
158  if (this->PPOpts->GeneratePreamble)
159  PreambleConditionalStack.startRecording();
160 }
161 
163  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
164 
165  IncludeMacroStack.clear();
166 
167  // Destroy any macro definitions.
168  while (MacroInfoChain *I = MIChainHead) {
169  MIChainHead = I->Next;
170  I->~MacroInfoChain();
171  }
172 
173  // Free any cached macro expanders.
174  // This populates MacroArgCache, so all TokenLexers need to be destroyed
175  // before the code below that frees up the MacroArgCache list.
176  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
177  CurTokenLexer.reset();
178 
179  // Free any cached MacroArgs.
180  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
181  ArgList = ArgList->deallocate();
182 
183  // Delete the header search info, if we own it.
184  if (OwnsHeaderSearch)
185  delete &HeaderInfo;
186 }
187 
189  const TargetInfo *AuxTarget) {
190  assert((!this->Target || this->Target == &Target) &&
191  "Invalid override of target information");
192  this->Target = &Target;
193 
194  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
195  "Invalid override of aux target information.");
196  this->AuxTarget = AuxTarget;
197 
198  // Initialize information about built-ins.
199  BuiltinInfo.InitializeTarget(Target, AuxTarget);
200  HeaderInfo.setTarget(Target);
201 
202  // Populate the identifier table with info about keywords for the current language.
203  Identifiers.AddKeywords(LangOpts);
204 }
205 
207  NumEnteredSourceFiles = 0;
208 
209  // Reset pragmas
210  PragmaHandlersBackup = std::move(PragmaHandlers);
211  PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
212  RegisterBuiltinPragmas();
213 
214  // Reset PredefinesFileID
215  PredefinesFileID = FileID();
216 }
217 
219  NumEnteredSourceFiles = 1;
220 
221  PragmaHandlers = std::move(PragmaHandlersBackup);
222 }
223 
224 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
225  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
226  << getSpelling(Tok) << "'";
227 
228  if (!DumpFlags) return;
229 
230  llvm::errs() << "\t";
231  if (Tok.isAtStartOfLine())
232  llvm::errs() << " [StartOfLine]";
233  if (Tok.hasLeadingSpace())
234  llvm::errs() << " [LeadingSpace]";
235  if (Tok.isExpandDisabled())
236  llvm::errs() << " [ExpandDisabled]";
237  if (Tok.needsCleaning()) {
238  const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
239  llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
240  << "']";
241  }
242 
243  llvm::errs() << "\tLoc=<";
244  DumpLocation(Tok.getLocation());
245  llvm::errs() << ">";
246 }
247 
249  Loc.print(llvm::errs(), SourceMgr);
250 }
251 
252 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
253  llvm::errs() << "MACRO: ";
254  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
256  llvm::errs() << " ";
257  }
258  llvm::errs() << "\n";
259 }
260 
262  llvm::errs() << "\n*** Preprocessor Stats:\n";
263  llvm::errs() << NumDirectives << " directives found:\n";
264  llvm::errs() << " " << NumDefined << " #define.\n";
265  llvm::errs() << " " << NumUndefined << " #undef.\n";
266  llvm::errs() << " #include/#include_next/#import:\n";
267  llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
268  llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
269  llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
270  llvm::errs() << " " << NumElse << " #else/#elif.\n";
271  llvm::errs() << " " << NumEndif << " #endif.\n";
272  llvm::errs() << " " << NumPragma << " #pragma.\n";
273  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
274 
275  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
276  << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
277  << NumFastMacroExpanded << " on the fast path.\n";
278  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
279  << " token paste (##) operations performed, "
280  << NumFastTokenPaste << " on the fast path.\n";
281 
282  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
283 
284  llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
285  llvm::errs() << "\n Macro Expanded Tokens: "
286  << llvm::capacity_in_bytes(MacroExpandedTokens);
287  llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
288  // FIXME: List information for all submodules.
289  llvm::errs() << "\n Macros: "
290  << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
291  llvm::errs() << "\n #pragma push_macro Info: "
292  << llvm::capacity_in_bytes(PragmaPushMacroInfo);
293  llvm::errs() << "\n Poison Reasons: "
294  << llvm::capacity_in_bytes(PoisonReasons);
295  llvm::errs() << "\n Comment Handlers: "
296  << llvm::capacity_in_bytes(CommentHandlers) << "\n";
297 }
298 
300 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
301  if (IncludeExternalMacros && ExternalSource &&
302  !ReadMacrosFromExternalSource) {
303  ReadMacrosFromExternalSource = true;
304  ExternalSource->ReadDefinedMacros();
305  }
306 
307  // Make sure we cover all macros in visible modules.
308  for (const ModuleMacro &Macro : ModuleMacros)
309  CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
310 
311  return CurSubmoduleState->Macros.begin();
312 }
313 
315  return BP.getTotalMemory()
316  + llvm::capacity_in_bytes(MacroExpandedTokens)
317  + Predefines.capacity() /* Predefines buffer. */
318  // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
319  // and ModuleMacros.
320  + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
321  + llvm::capacity_in_bytes(PragmaPushMacroInfo)
322  + llvm::capacity_in_bytes(PoisonReasons)
323  + llvm::capacity_in_bytes(CommentHandlers);
324 }
325 
327 Preprocessor::macro_end(bool IncludeExternalMacros) const {
328  if (IncludeExternalMacros && ExternalSource &&
329  !ReadMacrosFromExternalSource) {
330  ReadMacrosFromExternalSource = true;
331  ExternalSource->ReadDefinedMacros();
332  }
333 
334  return CurSubmoduleState->Macros.end();
335 }
336 
337 /// Compares macro tokens with a specified token value sequence.
338 static bool MacroDefinitionEquals(const MacroInfo *MI,
339  ArrayRef<TokenValue> Tokens) {
340  return Tokens.size() == MI->getNumTokens() &&
341  std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
342 }
343 
345  SourceLocation Loc,
346  ArrayRef<TokenValue> Tokens) const {
347  SourceLocation BestLocation;
348  StringRef BestSpelling;
350  I != E; ++I) {
352  Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
353  if (!Def || !Def.getMacroInfo())
354  continue;
355  if (!Def.getMacroInfo()->isObjectLike())
356  continue;
357  if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
358  continue;
359  SourceLocation Location = Def.getLocation();
360  // Choose the macro defined latest.
361  if (BestLocation.isInvalid() ||
362  (Location.isValid() &&
363  SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
364  BestLocation = Location;
365  BestSpelling = I->first->getName();
366  }
367  }
368  return BestSpelling;
369 }
370 
372  if (CurLexer)
373  CurLexerKind = CLK_Lexer;
374  else if (CurTokenLexer)
375  CurLexerKind = CLK_TokenLexer;
376  else
377  CurLexerKind = CLK_CachingLexer;
378 }
379 
381  unsigned CompleteLine,
382  unsigned CompleteColumn) {
383  assert(File);
384  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
385  assert(!CodeCompletionFile && "Already set");
386 
387  using llvm::MemoryBuffer;
388 
389  // Load the actual file's contents.
390  bool Invalid = false;
391  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
392  if (Invalid)
393  return true;
394 
395  // Find the byte position of the truncation point.
396  const char *Position = Buffer->getBufferStart();
397  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
398  for (; *Position; ++Position) {
399  if (*Position != '\r' && *Position != '\n')
400  continue;
401 
402  // Eat \r\n or \n\r as a single line.
403  if ((Position[1] == '\r' || Position[1] == '\n') &&
404  Position[0] != Position[1])
405  ++Position;
406  ++Position;
407  break;
408  }
409  }
410 
411  Position += CompleteColumn - 1;
412 
413  // If pointing inside the preamble, adjust the position at the beginning of
414  // the file after the preamble.
415  if (SkipMainFilePreamble.first &&
416  SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
417  if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
418  Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
419  }
420 
421  if (Position > Buffer->getBufferEnd())
422  Position = Buffer->getBufferEnd();
423 
424  CodeCompletionFile = File;
425  CodeCompletionOffset = Position - Buffer->getBufferStart();
426 
427  auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
428  Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
429  char *NewBuf = NewBuffer->getBufferStart();
430  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
431  *NewPos = '\0';
432  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
433  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
434 
435  return false;
436 }
437 
438 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
439  bool IsAngled) {
440  if (CodeComplete)
441  CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
443 }
444 
446  if (CodeComplete)
447  CodeComplete->CodeCompleteNaturalLanguage();
449 }
450 
451 /// getSpelling - This method is used to get the spelling of a token into a
452 /// SmallVector. Note that the returned StringRef may not point to the
453 /// supplied buffer if a copy can be avoided.
455  SmallVectorImpl<char> &Buffer,
456  bool *Invalid) const {
457  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
458  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
459  // Try the fast path.
460  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
461  return II->getName();
462  }
463 
464  // Resize the buffer if we need to copy into it.
465  if (Tok.needsCleaning())
466  Buffer.resize(Tok.getLength());
467 
468  const char *Ptr = Buffer.data();
469  unsigned Len = getSpelling(Tok, Ptr, Invalid);
470  return StringRef(Ptr, Len);
471 }
472 
473 /// CreateString - Plop the specified string into a scratch buffer and return a
474 /// location for it. If specified, the source location provides a source
475 /// location for the token.
476 void Preprocessor::CreateString(StringRef Str, Token &Tok,
477  SourceLocation ExpansionLocStart,
478  SourceLocation ExpansionLocEnd) {
479  Tok.setLength(Str.size());
480 
481  const char *DestPtr;
482  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
483 
484  if (ExpansionLocStart.isValid())
485  Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
486  ExpansionLocEnd, Str.size());
487  Tok.setLocation(Loc);
488 
489  // If this is a raw identifier or a literal token, set the pointer data.
490  if (Tok.is(tok::raw_identifier))
491  Tok.setRawIdentifierData(DestPtr);
492  else if (Tok.isLiteral())
493  Tok.setLiteralData(DestPtr);
494 }
495 
497  auto &SM = getSourceManager();
498  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
499  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
500  bool Invalid = false;
501  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
502  if (Invalid)
503  return SourceLocation();
504 
505  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
506  const char *DestPtr;
507  SourceLocation Spelling =
508  ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
509  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
510 }
511 
513  if (!getLangOpts().isCompilingModule())
514  return nullptr;
515 
516  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
517 }
518 
519 //===----------------------------------------------------------------------===//
520 // Preprocessor Initialization Methods
521 //===----------------------------------------------------------------------===//
522 
523 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
524 /// which implicitly adds the builtin defines etc.
526  // We do not allow the preprocessor to reenter the main file. Doing so will
527  // cause FileID's to accumulate information from both runs (e.g. #line
528  // information) and predefined macros aren't guaranteed to be set properly.
529  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
530  FileID MainFileID = SourceMgr.getMainFileID();
531 
532  // If MainFileID is loaded it means we loaded an AST file, no need to enter
533  // a main file.
534  if (!SourceMgr.isLoadedFileID(MainFileID)) {
535  // Enter the main file source buffer.
536  EnterSourceFile(MainFileID, nullptr, SourceLocation());
537 
538  // If we've been asked to skip bytes in the main file (e.g., as part of a
539  // precompiled preamble), do so now.
540  if (SkipMainFilePreamble.first > 0)
541  CurLexer->SetByteOffset(SkipMainFilePreamble.first,
542  SkipMainFilePreamble.second);
543 
544  // Tell the header info that the main file was entered. If the file is later
545  // #imported, it won't be re-entered.
546  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
547  HeaderInfo.IncrementIncludeCount(FE);
548  }
549 
550  // Preprocess Predefines to populate the initial preprocessor state.
551  std::unique_ptr<llvm::MemoryBuffer> SB =
552  llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
553  assert(SB && "Cannot create predefined source buffer");
554  FileID FID = SourceMgr.createFileID(std::move(SB));
555  assert(FID.isValid() && "Could not create FileID for predefines?");
556  setPredefinesFileID(FID);
557 
558  // Start parsing the predefines.
559  EnterSourceFile(FID, nullptr, SourceLocation());
560 
561  if (!PPOpts->PCHThroughHeader.empty()) {
562  // Lookup and save the FileID for the through header. If it isn't found
563  // in the search path, it's a fatal error.
564  const DirectoryLookup *CurDir;
565  const FileEntry *File = LookupFile(
566  SourceLocation(), PPOpts->PCHThroughHeader,
567  /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, CurDir,
568  /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
569  /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr);
570  if (!File) {
571  Diag(SourceLocation(), diag::err_pp_through_header_not_found)
572  << PPOpts->PCHThroughHeader;
573  return;
574  }
575  setPCHThroughHeaderFileID(
576  SourceMgr.createFileID(File, SourceLocation(), SrcMgr::C_User));
577  }
578 
579  // Skip tokens from the Predefines and if needed the main file.
580  if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
581  (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
583 }
584 
585 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
586  assert(PCHThroughHeaderFileID.isInvalid() &&
587  "PCHThroughHeaderFileID already set!");
588  PCHThroughHeaderFileID = FID;
589 }
590 
592  assert(PCHThroughHeaderFileID.isValid() &&
593  "Invalid PCH through header FileID");
594  return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
595 }
596 
598  return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
599  PCHThroughHeaderFileID.isValid();
600 }
601 
603  return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
604  PCHThroughHeaderFileID.isValid();
605 }
606 
608  return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
609 }
610 
612  return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
613 }
614 
615 /// Skip tokens until after the #include of the through header or
616 /// until after a #pragma hdrstop is seen. Tokens in the predefines file
617 /// and the main file may be skipped. If the end of the predefines file
618 /// is reached, skipping continues into the main file. If the end of the
619 /// main file is reached, it's a fatal error.
621  bool ReachedMainFileEOF = false;
622  bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
623  bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
624  Token Tok;
625  while (true) {
626  bool InPredefines = (CurLexer->getFileID() == getPredefinesFileID());
627  CurLexer->Lex(Tok);
628  if (Tok.is(tok::eof) && !InPredefines) {
629  ReachedMainFileEOF = true;
630  break;
631  }
632  if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
633  break;
634  if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
635  break;
636  }
637  if (ReachedMainFileEOF) {
638  if (UsingPCHThroughHeader)
639  Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
640  << PPOpts->PCHThroughHeader << 1;
641  else if (!PPOpts->PCHWithHdrStopCreate)
642  Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
643  }
644 }
645 
646 void Preprocessor::replayPreambleConditionalStack() {
647  // Restore the conditional stack from the preamble, if there is one.
648  if (PreambleConditionalStack.isReplaying()) {
649  assert(CurPPLexer &&
650  "CurPPLexer is null when calling replayPreambleConditionalStack.");
651  CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
652  PreambleConditionalStack.doneReplaying();
653  if (PreambleConditionalStack.reachedEOFWhileSkipping())
654  SkipExcludedConditionalBlock(
655  PreambleConditionalStack.SkipInfo->HashTokenLoc,
656  PreambleConditionalStack.SkipInfo->IfTokenLoc,
657  PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
658  PreambleConditionalStack.SkipInfo->FoundElse,
659  PreambleConditionalStack.SkipInfo->ElseLoc);
660  }
661 }
662 
664  // Notify the client that we reached the end of the source file.
665  if (Callbacks)
666  Callbacks->EndOfMainFile();
667 }
668 
669 //===----------------------------------------------------------------------===//
670 // Lexer Event Handling.
671 //===----------------------------------------------------------------------===//
672 
673 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
674 /// identifier information for the token and install it into the token,
675 /// updating the token kind accordingly.
677  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
678 
679  // Look up this token, see if it is a macro, or if it is a language keyword.
680  IdentifierInfo *II;
681  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
682  // No cleaning needed, just use the characters from the lexed buffer.
683  II = getIdentifierInfo(Identifier.getRawIdentifier());
684  } else {
685  // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
686  SmallString<64> IdentifierBuffer;
687  StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
688 
689  if (Identifier.hasUCN()) {
690  SmallString<64> UCNIdentifierBuffer;
691  expandUCNs(UCNIdentifierBuffer, CleanedStr);
692  II = getIdentifierInfo(UCNIdentifierBuffer);
693  } else {
694  II = getIdentifierInfo(CleanedStr);
695  }
696  }
697 
698  // Update the token info (identifier info and appropriate token kind).
699  Identifier.setIdentifierInfo(II);
700  if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
702  Identifier.setKind(tok::identifier);
703  else
704  Identifier.setKind(II->getTokenID());
705 
706  return II;
707 }
708 
709 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
710  PoisonReasons[II] = DiagID;
711 }
712 
714  assert(Ident__exception_code && Ident__exception_info);
715  assert(Ident___exception_code && Ident___exception_info);
716  Ident__exception_code->setIsPoisoned(Poison);
717  Ident___exception_code->setIsPoisoned(Poison);
718  Ident_GetExceptionCode->setIsPoisoned(Poison);
719  Ident__exception_info->setIsPoisoned(Poison);
720  Ident___exception_info->setIsPoisoned(Poison);
721  Ident_GetExceptionInfo->setIsPoisoned(Poison);
722  Ident__abnormal_termination->setIsPoisoned(Poison);
723  Ident___abnormal_termination->setIsPoisoned(Poison);
724  Ident_AbnormalTermination->setIsPoisoned(Poison);
725 }
726 
728  assert(Identifier.getIdentifierInfo() &&
729  "Can't handle identifiers without identifier info!");
730  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
731  PoisonReasons.find(Identifier.getIdentifierInfo());
732  if(it == PoisonReasons.end())
733  Diag(Identifier, diag::err_pp_used_poisoned_id);
734  else
735  Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
736 }
737 
738 /// Returns a diagnostic message kind for reporting a future keyword as
739 /// appropriate for the identifier and specified language.
741  const LangOptions &LangOpts) {
742  assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
743 
744  if (LangOpts.CPlusPlus)
745  return llvm::StringSwitch<diag::kind>(II.getName())
746 #define CXX11_KEYWORD(NAME, FLAGS) \
747  .Case(#NAME, diag::warn_cxx11_keyword)
748 #define CXX2A_KEYWORD(NAME, FLAGS) \
749  .Case(#NAME, diag::warn_cxx2a_keyword)
750 #include "clang/Basic/TokenKinds.def"
751  ;
752 
753  llvm_unreachable(
754  "Keyword not known to come from a newer Standard or proposed Standard");
755 }
756 
757 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
758  assert(II.isOutOfDate() && "not out of date");
760 }
761 
762 /// HandleIdentifier - This callback is invoked when the lexer reads an
763 /// identifier. This callback looks up the identifier in the map and/or
764 /// potentially macro expands it or turns it into a named token (like 'for').
765 ///
766 /// Note that callers of this method are guarded by checking the
767 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
768 /// IdentifierInfo methods that compute these properties will need to change to
769 /// match.
771  assert(Identifier.getIdentifierInfo() &&
772  "Can't handle identifiers without identifier info!");
773 
774  IdentifierInfo &II = *Identifier.getIdentifierInfo();
775 
776  // If the information about this identifier is out of date, update it from
777  // the external source.
778  // We have to treat __VA_ARGS__ in a special way, since it gets
779  // serialized with isPoisoned = true, but our preprocessor may have
780  // unpoisoned it if we're defining a C99 macro.
781  if (II.isOutOfDate()) {
782  bool CurrentIsPoisoned = false;
783  const bool IsSpecialVariadicMacro =
784  &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
785  if (IsSpecialVariadicMacro)
786  CurrentIsPoisoned = II.isPoisoned();
787 
788  updateOutOfDateIdentifier(II);
789  Identifier.setKind(II.getTokenID());
790 
791  if (IsSpecialVariadicMacro)
792  II.setIsPoisoned(CurrentIsPoisoned);
793  }
794 
795  // If this identifier was poisoned, and if it was not produced from a macro
796  // expansion, emit an error.
797  if (II.isPoisoned() && CurPPLexer) {
798  HandlePoisonedIdentifier(Identifier);
799  }
800 
801  // If this is a macro to be expanded, do it.
802  if (MacroDefinition MD = getMacroDefinition(&II)) {
803  auto *MI = MD.getMacroInfo();
804  assert(MI && "macro definition with no macro info?");
805  if (!DisableMacroExpansion) {
806  if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
807  // C99 6.10.3p10: If the preprocessing token immediately after the
808  // macro name isn't a '(', this macro should not be expanded.
809  if (!MI->isFunctionLike() || isNextPPTokenLParen())
810  return HandleMacroExpandedIdentifier(Identifier, MD);
811  } else {
812  // C99 6.10.3.4p2 says that a disabled macro may never again be
813  // expanded, even if it's in a context where it could be expanded in the
814  // future.
815  Identifier.setFlag(Token::DisableExpand);
816  if (MI->isObjectLike() || isNextPPTokenLParen())
817  Diag(Identifier, diag::pp_disabled_macro_expansion);
818  }
819  }
820  }
821 
822  // If this identifier is a keyword in a newer Standard or proposed Standard,
823  // produce a warning. Don't warn if we're not considering macro expansion,
824  // since this identifier might be the name of a macro.
825  // FIXME: This warning is disabled in cases where it shouldn't be, like
826  // "#define constexpr constexpr", "int constexpr;"
827  if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
828  Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
829  << II.getName();
830  // Don't diagnose this keyword again in this translation unit.
831  II.setIsFutureCompatKeyword(false);
832  }
833 
834  // If this is an extension token, diagnose its use.
835  // We avoid diagnosing tokens that originate from macro definitions.
836  // FIXME: This warning is disabled in cases where it shouldn't be,
837  // like "#define TY typeof", "TY(1) x".
838  if (II.isExtensionToken() && !DisableMacroExpansion)
839  Diag(Identifier, diag::ext_token_used);
840 
841  // If this is the 'import' contextual keyword following an '@', note
842  // that the next token indicates a module name.
843  //
844  // Note that we do not treat 'import' as a contextual
845  // keyword when we're in a caching lexer, because caching lexers only get
846  // used in contexts where import declarations are disallowed.
847  //
848  // Likewise if this is the C++ Modules TS import keyword.
849  if (((LastTokenWasAt && II.isModulesImport()) ||
850  Identifier.is(tok::kw_import)) &&
851  !InMacroArgs && !DisableMacroExpansion &&
852  (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
853  CurLexerKind != CLK_CachingLexer) {
854  ModuleImportLoc = Identifier.getLocation();
855  ModuleImportPath.clear();
856  ModuleImportExpectsIdentifier = true;
857  CurLexerKind = CLK_LexAfterModuleImport;
858  }
859  return true;
860 }
861 
863  // We loop here until a lex function returns a token; this avoids recursion.
864  bool ReturnedToken;
865  do {
866  switch (CurLexerKind) {
867  case CLK_Lexer:
868  ReturnedToken = CurLexer->Lex(Result);
869  break;
870  case CLK_TokenLexer:
871  ReturnedToken = CurTokenLexer->Lex(Result);
872  break;
873  case CLK_CachingLexer:
874  CachingLex(Result);
875  ReturnedToken = true;
876  break;
877  case CLK_LexAfterModuleImport:
878  LexAfterModuleImport(Result);
879  ReturnedToken = true;
880  break;
881  }
882  } while (!ReturnedToken);
883 
884  if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
885  // Remember the identifier before code completion token.
888  // Set IdenfitierInfo to null to avoid confusing code that handles both
889  // identifiers and completion tokens.
890  Result.setIdentifierInfo(nullptr);
891  }
892 
893  LastTokenWasAt = Result.is(tok::at);
894 }
895 
896 /// Lex a token following the 'import' contextual keyword.
897 ///
899  // Figure out what kind of lexer we actually have.
901 
902  // Lex the next token.
903  Lex(Result);
904 
905  // The token sequence
906  //
907  // import identifier (. identifier)*
908  //
909  // indicates a module import directive. We already saw the 'import'
910  // contextual keyword, so now we're looking for the identifiers.
911  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
912  // We expected to see an identifier here, and we did; continue handling
913  // identifiers.
914  ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
915  Result.getLocation()));
916  ModuleImportExpectsIdentifier = false;
917  CurLexerKind = CLK_LexAfterModuleImport;
918  return;
919  }
920 
921  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
922  // see the next identifier. (We can also see a '[[' that begins an
923  // attribute-specifier-seq here under the C++ Modules TS.)
924  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
925  ModuleImportExpectsIdentifier = true;
926  CurLexerKind = CLK_LexAfterModuleImport;
927  return;
928  }
929 
930  // If we have a non-empty module path, load the named module.
931  if (!ModuleImportPath.empty()) {
932  // Under the Modules TS, the dot is just part of the module name, and not
933  // a real hierarchy separator. Flatten such module names now.
934  //
935  // FIXME: Is this the right level to be performing this transformation?
936  std::string FlatModuleName;
937  if (getLangOpts().ModulesTS) {
938  for (auto &Piece : ModuleImportPath) {
939  if (!FlatModuleName.empty())
940  FlatModuleName += ".";
941  FlatModuleName += Piece.first->getName();
942  }
943  SourceLocation FirstPathLoc = ModuleImportPath[0].second;
944  ModuleImportPath.clear();
945  ModuleImportPath.push_back(
946  std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
947  }
948 
949  Module *Imported = nullptr;
950  if (getLangOpts().Modules) {
951  Imported = TheModuleLoader.loadModule(ModuleImportLoc,
952  ModuleImportPath,
954  /*IsIncludeDirective=*/false);
955  if (Imported)
956  makeModuleVisible(Imported, ModuleImportLoc);
957  }
958  if (Callbacks && (getLangOpts().Modules || getLangOpts().DebuggerSupport))
959  Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
960  }
961 }
962 
964  CurSubmoduleState->VisibleModules.setVisible(
965  M, Loc, [](Module *) {},
966  [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
967  // FIXME: Include the path in the diagnostic.
968  // FIXME: Include the import location for the conflicting module.
969  Diag(ModuleImportLoc, diag::warn_module_conflict)
970  << Path[0]->getFullModuleName()
971  << Conflict->getFullModuleName()
972  << Message;
973  });
974 
975  // Add this module to the imports list of the currently-built submodule.
976  if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
977  BuildingSubmoduleStack.back().M->Imports.insert(M);
978 }
979 
981  const char *DiagnosticTag,
982  bool AllowMacroExpansion) {
983  // We need at least one string literal.
984  if (Result.isNot(tok::string_literal)) {
985  Diag(Result, diag::err_expected_string_literal)
986  << /*Source='in...'*/0 << DiagnosticTag;
987  return false;
988  }
989 
990  // Lex string literal tokens, optionally with macro expansion.
991  SmallVector<Token, 4> StrToks;
992  do {
993  StrToks.push_back(Result);
994 
995  if (Result.hasUDSuffix())
996  Diag(Result, diag::err_invalid_string_udl);
997 
998  if (AllowMacroExpansion)
999  Lex(Result);
1000  else
1001  LexUnexpandedToken(Result);
1002  } while (Result.is(tok::string_literal));
1003 
1004  // Concatenate and parse the strings.
1005  StringLiteralParser Literal(StrToks, *this);
1006  assert(Literal.isAscii() && "Didn't allow wide strings in");
1007 
1008  if (Literal.hadError)
1009  return false;
1010 
1011  if (Literal.Pascal) {
1012  Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1013  << /*Source='in...'*/0 << DiagnosticTag;
1014  return false;
1015  }
1016 
1017  String = Literal.GetString();
1018  return true;
1019 }
1020 
1022  assert(Tok.is(tok::numeric_constant));
1023  SmallString<8> IntegerBuffer;
1024  bool NumberInvalid = false;
1025  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1026  if (NumberInvalid)
1027  return false;
1028  NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
1029  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1030  return false;
1031  llvm::APInt APVal(64, 0);
1032  if (Literal.GetIntegerValue(APVal))
1033  return false;
1034  Lex(Tok);
1035  Value = APVal.getLimitedValue();
1036  return true;
1037 }
1038 
1040  assert(Handler && "NULL comment handler");
1041  assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
1042  CommentHandlers.end() && "Comment handler already registered");
1043  CommentHandlers.push_back(Handler);
1044 }
1045 
1047  std::vector<CommentHandler *>::iterator Pos =
1048  std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
1049  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1050  CommentHandlers.erase(Pos);
1051 }
1052 
1054  bool AnyPendingTokens = false;
1055  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1056  HEnd = CommentHandlers.end();
1057  H != HEnd; ++H) {
1058  if ((*H)->HandleComment(*this, Comment))
1059  AnyPendingTokens = true;
1060  }
1061  if (!AnyPendingTokens || getCommentRetentionState())
1062  return false;
1063  Lex(result);
1064  return true;
1065 }
1066 
1067 ModuleLoader::~ModuleLoader() = default;
1068 
1070 
1072 
1074  if (Record)
1075  return;
1076 
1077  Record = new PreprocessingRecord(getSourceManager());
1078  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1079 }
StringRef Identifier
Definition: Format.cpp:1635
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void FinalizeForModelFile()
Cleanup after model file parsing.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:94
Defines the SourceManager interface.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Defines the clang::Module class, which describes a module in the source code.
Module * getCurrentModule()
Retrieves the module that we&#39;re currently building, if any.
Defines the FileSystemStatCache interface.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:367
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void LexAfterModuleImport(Token &Result)
Lex a token following the &#39;import&#39; contextual keyword.
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:233
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
Manage memory buffers across multiple users.
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:242
tok::TokenKind getKind() const
Definition: Token.h:89
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:112
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
void print(raw_ostream &OS, const SourceManager &SM) const
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:206
Definition: Format.h:2071
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, MemoryBufferCache &PCMCache, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void setKind(tok::TokenKind K)
Definition: Token.h:90
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive...
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:814
Describes a module or submodule.
Definition: Module.h:64
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
virtual ~ModuleLoader()
bool isInvalid() const
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:820
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:235
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
Definition: Token.h:290
const FormatToken & Tok
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:148
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
bool HandleComment(Token &result, SourceRange Comment)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:147
Provides lookups to, and iteration over, IdentiferInfo objects.
bool isValid() const
Exposes information about the current target.
Definition: TargetInfo.h:53
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
void makeModuleVisible(Module *M, SourceLocation Loc)
SourceLocation getLocation() const
Definition: MacroInfo.h:462
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:123
Defines the clang::Preprocessor interface.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
size_t getTotalMemory() const
bool getCommentRetentionState() const
Definition: Preprocessor.h:854
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isObjectLike() const
Definition: MacroInfo.h:200
void IncrementIncludeCount(const FileEntry *File)
Increment the count for the number of times the specified FileEntry has been entered.
Definition: HeaderSearch.h:454
The result type of a method or function.
StringRef GetString() const
const SourceManager & SM
Definition: Format.cpp:1489
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:818
void DumpMacro(const MacroInfo &MI) const
void overrideFileContents(const FileEntry *SourceFile, llvm::MemoryBuffer *Buffer, bool DoNotFree)
Override the contents of the given source file by providing an already-allocated buffer.
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:832
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6...
Definition: Token.h:273
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:132
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:939
All of the names in this module are hidden.
Definition: Module.h:268
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:176
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:185
void Lex(Token &Result)
Lex the next token for this preprocessor.
llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:265
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:95
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:233
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
FileID getMainFileID() const
Returns the FileID of the main source file.
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:126
void PoisonSEHIdentifiers(bool Poison=true)
void setLiteralData(const char *Ptr)
Definition: Token.h:218
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:89
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
Module * lookupModule(StringRef ModuleName, bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
#define CXX11_KEYWORD(NAME, FLAGS)
Defines the clang::SourceLocation class and associated facilities.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:899
StringRef getRawIdentifier() const
getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode), returns a reference to the text substring in the buffer if known.
Definition: Token.h:202
SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc, SourceLocation TokenStart, SourceLocation TokenEnd)
Return a new SourceLocation that encodes that the token starting at TokenStart ends prematurely at To...
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:361
void DumpLocation(SourceLocation Loc) const
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Definition: DiagnosticIDs.h:60
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:24
Defines the clang::TargetInfo interface.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:282
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber...
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void setLocation(SourceLocation L)
Definition: Token.h:131
#define true
Definition: stdbool.h:32
A trivial tuple used to represent a source range.
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:293
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Defines the PreprocessorLexer interface.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:24
void InitializeTarget(const TargetInfo &Target, const TargetInfo *AuxTarget)
Perform target-specific initialization.
Definition: Builtins.cpp:42
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:269
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:907
This class handles loading and caching of source files into memory.
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
SourceLocation getEndLoc() const
Definition: Token.h:150