clang  9.0.0svn
Preprocessor.cpp
Go to the documentation of this file.
1 //===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Preprocessor interface.
10 //
11 //===----------------------------------------------------------------------===//
12 //
13 // Options to support:
14 // -H - Print the name of each header file used.
15 // -d[DNI] - Dump various things.
16 // -fworking-directory - #line's with preprocessor's working dir.
17 // -fpreprocessed
18 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19 // -W*
20 // -w
21 //
22 // Messages to emit:
23 // "Multiple include guards may be useful for:\n"
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "clang/Lex/Preprocessor.h"
31 #include "clang/Basic/LLVM.h"
33 #include "clang/Basic/Module.h"
36 #include "clang/Basic/TargetInfo.h"
39 #include "clang/Lex/HeaderSearch.h"
41 #include "clang/Lex/Lexer.h"
43 #include "clang/Lex/MacroArgs.h"
44 #include "clang/Lex/MacroInfo.h"
45 #include "clang/Lex/ModuleLoader.h"
46 #include "clang/Lex/Pragma.h"
51 #include "clang/Lex/Token.h"
52 #include "clang/Lex/TokenLexer.h"
53 #include "llvm/ADT/APInt.h"
54 #include "llvm/ADT/ArrayRef.h"
55 #include "llvm/ADT/DenseMap.h"
56 #include "llvm/ADT/SmallString.h"
57 #include "llvm/ADT/SmallVector.h"
58 #include "llvm/ADT/STLExtras.h"
59 #include "llvm/ADT/StringRef.h"
60 #include "llvm/ADT/StringSwitch.h"
61 #include "llvm/Support/Capacity.h"
62 #include "llvm/Support/ErrorHandling.h"
63 #include "llvm/Support/MemoryBuffer.h"
64 #include "llvm/Support/raw_ostream.h"
65 #include <algorithm>
66 #include <cassert>
67 #include <memory>
68 #include <string>
69 #include <utility>
70 #include <vector>
71 
72 using namespace clang;
73 
74 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
75 
77 
78 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
79  DiagnosticsEngine &diags, LangOptions &opts,
80  SourceManager &SM, HeaderSearch &Headers,
81  ModuleLoader &TheModuleLoader,
82  IdentifierInfoLookup *IILookup, bool OwnsHeaders,
83  TranslationUnitKind TUKind)
84  : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
85  FileMgr(Headers.getFileMgr()), SourceMgr(SM),
86  ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
87  TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
88  // As the language options may have not been loaded yet (when
89  // deserializing an ASTUnit), adding keywords to the identifier table is
90  // deferred to Preprocessor::Initialize().
91  Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
92  TUKind(TUKind), SkipMainFilePreamble(0, true),
93  CurSubmoduleState(&NullSubmoduleState) {
94  OwnsHeaderSearch = OwnsHeaders;
95 
96  // Default to discarding comments.
97  KeepComments = false;
98  KeepMacroComments = false;
99  SuppressIncludeNotFoundError = false;
100 
101  // Macro expansion is enabled.
102  DisableMacroExpansion = false;
103  MacroExpansionInDirectivesOverride = false;
104  InMacroArgs = false;
105  ArgMacro = nullptr;
106  InMacroArgPreExpansion = false;
107  NumCachedTokenLexers = 0;
108  PragmasEnabled = true;
109  ParsingIfOrElifDirective = false;
110  PreprocessedOutput = false;
111 
112  // We haven't read anything from the external source.
113  ReadMacrosFromExternalSource = false;
114 
115  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
116  // a macro. They get unpoisoned where it is allowed.
117  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
118  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
119  if (getLangOpts().CPlusPlus2a) {
120  (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
121  SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
122  } else {
123  Ident__VA_OPT__ = nullptr;
124  }
125 
126  // Initialize the pragma handlers.
127  RegisterBuiltinPragmas();
128 
129  // Initialize builtin macros like __LINE__ and friends.
130  RegisterBuiltinMacros();
131 
132  if(LangOpts.Borland) {
133  Ident__exception_info = getIdentifierInfo("_exception_info");
134  Ident___exception_info = getIdentifierInfo("__exception_info");
135  Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
136  Ident__exception_code = getIdentifierInfo("_exception_code");
137  Ident___exception_code = getIdentifierInfo("__exception_code");
138  Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
139  Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
140  Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
141  Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
142  } else {
143  Ident__exception_info = Ident__exception_code = nullptr;
144  Ident__abnormal_termination = Ident___exception_info = nullptr;
145  Ident___exception_code = Ident___abnormal_termination = nullptr;
146  Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
147  Ident_AbnormalTermination = nullptr;
148  }
149 
150  // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
152  SkippingUntilPragmaHdrStop = true;
153 
154  // If using a PCH with a through header, start skipping tokens.
155  if (!this->PPOpts->PCHThroughHeader.empty() &&
156  !this->PPOpts->ImplicitPCHInclude.empty())
157  SkippingUntilPCHThroughHeader = true;
158 
159  if (this->PPOpts->GeneratePreamble)
160  PreambleConditionalStack.startRecording();
161 }
162 
164  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
165 
166  IncludeMacroStack.clear();
167 
168  // Destroy any macro definitions.
169  while (MacroInfoChain *I = MIChainHead) {
170  MIChainHead = I->Next;
171  I->~MacroInfoChain();
172  }
173 
174  // Free any cached macro expanders.
175  // This populates MacroArgCache, so all TokenLexers need to be destroyed
176  // before the code below that frees up the MacroArgCache list.
177  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
178  CurTokenLexer.reset();
179 
180  // Free any cached MacroArgs.
181  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
182  ArgList = ArgList->deallocate();
183 
184  // Delete the header search info, if we own it.
185  if (OwnsHeaderSearch)
186  delete &HeaderInfo;
187 }
188 
190  const TargetInfo *AuxTarget) {
191  assert((!this->Target || this->Target == &Target) &&
192  "Invalid override of target information");
193  this->Target = &Target;
194 
195  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
196  "Invalid override of aux target information.");
197  this->AuxTarget = AuxTarget;
198 
199  // Initialize information about built-ins.
200  BuiltinInfo.InitializeTarget(Target, AuxTarget);
201  HeaderInfo.setTarget(Target);
202 
203  // Populate the identifier table with info about keywords for the current language.
204  Identifiers.AddKeywords(LangOpts);
205 }
206 
208  NumEnteredSourceFiles = 0;
209 
210  // Reset pragmas
211  PragmaHandlersBackup = std::move(PragmaHandlers);
212  PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
213  RegisterBuiltinPragmas();
214 
215  // Reset PredefinesFileID
216  PredefinesFileID = FileID();
217 }
218 
220  NumEnteredSourceFiles = 1;
221 
222  PragmaHandlers = std::move(PragmaHandlersBackup);
223 }
224 
225 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
226  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
227  << getSpelling(Tok) << "'";
228 
229  if (!DumpFlags) return;
230 
231  llvm::errs() << "\t";
232  if (Tok.isAtStartOfLine())
233  llvm::errs() << " [StartOfLine]";
234  if (Tok.hasLeadingSpace())
235  llvm::errs() << " [LeadingSpace]";
236  if (Tok.isExpandDisabled())
237  llvm::errs() << " [ExpandDisabled]";
238  if (Tok.needsCleaning()) {
239  const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
240  llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
241  << "']";
242  }
243 
244  llvm::errs() << "\tLoc=<";
245  DumpLocation(Tok.getLocation());
246  llvm::errs() << ">";
247 }
248 
250  Loc.print(llvm::errs(), SourceMgr);
251 }
252 
253 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
254  llvm::errs() << "MACRO: ";
255  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
257  llvm::errs() << " ";
258  }
259  llvm::errs() << "\n";
260 }
261 
263  llvm::errs() << "\n*** Preprocessor Stats:\n";
264  llvm::errs() << NumDirectives << " directives found:\n";
265  llvm::errs() << " " << NumDefined << " #define.\n";
266  llvm::errs() << " " << NumUndefined << " #undef.\n";
267  llvm::errs() << " #include/#include_next/#import:\n";
268  llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
269  llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
270  llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
271  llvm::errs() << " " << NumElse << " #else/#elif.\n";
272  llvm::errs() << " " << NumEndif << " #endif.\n";
273  llvm::errs() << " " << NumPragma << " #pragma.\n";
274  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
275 
276  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
277  << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
278  << NumFastMacroExpanded << " on the fast path.\n";
279  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
280  << " token paste (##) operations performed, "
281  << NumFastTokenPaste << " on the fast path.\n";
282 
283  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
284 
285  llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
286  llvm::errs() << "\n Macro Expanded Tokens: "
287  << llvm::capacity_in_bytes(MacroExpandedTokens);
288  llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
289  // FIXME: List information for all submodules.
290  llvm::errs() << "\n Macros: "
291  << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
292  llvm::errs() << "\n #pragma push_macro Info: "
293  << llvm::capacity_in_bytes(PragmaPushMacroInfo);
294  llvm::errs() << "\n Poison Reasons: "
295  << llvm::capacity_in_bytes(PoisonReasons);
296  llvm::errs() << "\n Comment Handlers: "
297  << llvm::capacity_in_bytes(CommentHandlers) << "\n";
298 }
299 
301 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
302  if (IncludeExternalMacros && ExternalSource &&
303  !ReadMacrosFromExternalSource) {
304  ReadMacrosFromExternalSource = true;
305  ExternalSource->ReadDefinedMacros();
306  }
307 
308  // Make sure we cover all macros in visible modules.
309  for (const ModuleMacro &Macro : ModuleMacros)
310  CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
311 
312  return CurSubmoduleState->Macros.begin();
313 }
314 
316  return BP.getTotalMemory()
317  + llvm::capacity_in_bytes(MacroExpandedTokens)
318  + Predefines.capacity() /* Predefines buffer. */
319  // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
320  // and ModuleMacros.
321  + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
322  + llvm::capacity_in_bytes(PragmaPushMacroInfo)
323  + llvm::capacity_in_bytes(PoisonReasons)
324  + llvm::capacity_in_bytes(CommentHandlers);
325 }
326 
328 Preprocessor::macro_end(bool IncludeExternalMacros) const {
329  if (IncludeExternalMacros && ExternalSource &&
330  !ReadMacrosFromExternalSource) {
331  ReadMacrosFromExternalSource = true;
332  ExternalSource->ReadDefinedMacros();
333  }
334 
335  return CurSubmoduleState->Macros.end();
336 }
337 
338 /// Compares macro tokens with a specified token value sequence.
339 static bool MacroDefinitionEquals(const MacroInfo *MI,
340  ArrayRef<TokenValue> Tokens) {
341  return Tokens.size() == MI->getNumTokens() &&
342  std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
343 }
344 
346  SourceLocation Loc,
347  ArrayRef<TokenValue> Tokens) const {
348  SourceLocation BestLocation;
349  StringRef BestSpelling;
351  I != E; ++I) {
353  Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
354  if (!Def || !Def.getMacroInfo())
355  continue;
356  if (!Def.getMacroInfo()->isObjectLike())
357  continue;
358  if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
359  continue;
360  SourceLocation Location = Def.getLocation();
361  // Choose the macro defined latest.
362  if (BestLocation.isInvalid() ||
363  (Location.isValid() &&
364  SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
365  BestLocation = Location;
366  BestSpelling = I->first->getName();
367  }
368  }
369  return BestSpelling;
370 }
371 
373  if (CurLexer)
374  CurLexerKind = CLK_Lexer;
375  else if (CurTokenLexer)
376  CurLexerKind = CLK_TokenLexer;
377  else
378  CurLexerKind = CLK_CachingLexer;
379 }
380 
382  unsigned CompleteLine,
383  unsigned CompleteColumn) {
384  assert(File);
385  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
386  assert(!CodeCompletionFile && "Already set");
387 
388  using llvm::MemoryBuffer;
389 
390  // Load the actual file's contents.
391  bool Invalid = false;
392  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
393  if (Invalid)
394  return true;
395 
396  // Find the byte position of the truncation point.
397  const char *Position = Buffer->getBufferStart();
398  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
399  for (; *Position; ++Position) {
400  if (*Position != '\r' && *Position != '\n')
401  continue;
402 
403  // Eat \r\n or \n\r as a single line.
404  if ((Position[1] == '\r' || Position[1] == '\n') &&
405  Position[0] != Position[1])
406  ++Position;
407  ++Position;
408  break;
409  }
410  }
411 
412  Position += CompleteColumn - 1;
413 
414  // If pointing inside the preamble, adjust the position at the beginning of
415  // the file after the preamble.
416  if (SkipMainFilePreamble.first &&
417  SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
418  if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
419  Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
420  }
421 
422  if (Position > Buffer->getBufferEnd())
423  Position = Buffer->getBufferEnd();
424 
425  CodeCompletionFile = File;
426  CodeCompletionOffset = Position - Buffer->getBufferStart();
427 
428  auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
429  Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
430  char *NewBuf = NewBuffer->getBufferStart();
431  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
432  *NewPos = '\0';
433  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
434  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
435 
436  return false;
437 }
438 
439 void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
440  bool IsAngled) {
441  if (CodeComplete)
442  CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
444 }
445 
447  if (CodeComplete)
448  CodeComplete->CodeCompleteNaturalLanguage();
450 }
451 
452 /// getSpelling - This method is used to get the spelling of a token into a
453 /// SmallVector. Note that the returned StringRef may not point to the
454 /// supplied buffer if a copy can be avoided.
456  SmallVectorImpl<char> &Buffer,
457  bool *Invalid) const {
458  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
459  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
460  // Try the fast path.
461  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
462  return II->getName();
463  }
464 
465  // Resize the buffer if we need to copy into it.
466  if (Tok.needsCleaning())
467  Buffer.resize(Tok.getLength());
468 
469  const char *Ptr = Buffer.data();
470  unsigned Len = getSpelling(Tok, Ptr, Invalid);
471  return StringRef(Ptr, Len);
472 }
473 
474 /// CreateString - Plop the specified string into a scratch buffer and return a
475 /// location for it. If specified, the source location provides a source
476 /// location for the token.
477 void Preprocessor::CreateString(StringRef Str, Token &Tok,
478  SourceLocation ExpansionLocStart,
479  SourceLocation ExpansionLocEnd) {
480  Tok.setLength(Str.size());
481 
482  const char *DestPtr;
483  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
484 
485  if (ExpansionLocStart.isValid())
486  Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
487  ExpansionLocEnd, Str.size());
488  Tok.setLocation(Loc);
489 
490  // If this is a raw identifier or a literal token, set the pointer data.
491  if (Tok.is(tok::raw_identifier))
492  Tok.setRawIdentifierData(DestPtr);
493  else if (Tok.isLiteral())
494  Tok.setLiteralData(DestPtr);
495 }
496 
498  auto &SM = getSourceManager();
499  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
500  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
501  bool Invalid = false;
502  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
503  if (Invalid)
504  return SourceLocation();
505 
506  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
507  const char *DestPtr;
508  SourceLocation Spelling =
509  ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
510  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
511 }
512 
514  if (!getLangOpts().isCompilingModule())
515  return nullptr;
516 
517  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
518 }
519 
520 //===----------------------------------------------------------------------===//
521 // Preprocessor Initialization Methods
522 //===----------------------------------------------------------------------===//
523 
524 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
525 /// which implicitly adds the builtin defines etc.
527  // We do not allow the preprocessor to reenter the main file. Doing so will
528  // cause FileID's to accumulate information from both runs (e.g. #line
529  // information) and predefined macros aren't guaranteed to be set properly.
530  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
531  FileID MainFileID = SourceMgr.getMainFileID();
532 
533  // If MainFileID is loaded it means we loaded an AST file, no need to enter
534  // a main file.
535  if (!SourceMgr.isLoadedFileID(MainFileID)) {
536  // Enter the main file source buffer.
537  EnterSourceFile(MainFileID, nullptr, SourceLocation());
538 
539  // If we've been asked to skip bytes in the main file (e.g., as part of a
540  // precompiled preamble), do so now.
541  if (SkipMainFilePreamble.first > 0)
542  CurLexer->SetByteOffset(SkipMainFilePreamble.first,
543  SkipMainFilePreamble.second);
544 
545  // Tell the header info that the main file was entered. If the file is later
546  // #imported, it won't be re-entered.
547  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
548  HeaderInfo.IncrementIncludeCount(FE);
549  }
550 
551  // Preprocess Predefines to populate the initial preprocessor state.
552  std::unique_ptr<llvm::MemoryBuffer> SB =
553  llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
554  assert(SB && "Cannot create predefined source buffer");
555  FileID FID = SourceMgr.createFileID(std::move(SB));
556  assert(FID.isValid() && "Could not create FileID for predefines?");
557  setPredefinesFileID(FID);
558 
559  // Start parsing the predefines.
560  EnterSourceFile(FID, nullptr, SourceLocation());
561 
562  if (!PPOpts->PCHThroughHeader.empty()) {
563  // Lookup and save the FileID for the through header. If it isn't found
564  // in the search path, it's a fatal error.
565  const DirectoryLookup *CurDir;
566  const FileEntry *File = LookupFile(
567  SourceLocation(), PPOpts->PCHThroughHeader,
568  /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, CurDir,
569  /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
570  /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
571  /*IsFrameworkFound=*/nullptr);
572  if (!File) {
573  Diag(SourceLocation(), diag::err_pp_through_header_not_found)
574  << PPOpts->PCHThroughHeader;
575  return;
576  }
577  setPCHThroughHeaderFileID(
578  SourceMgr.createFileID(File, SourceLocation(), SrcMgr::C_User));
579  }
580 
581  // Skip tokens from the Predefines and if needed the main file.
582  if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
583  (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
585 }
586 
587 void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
588  assert(PCHThroughHeaderFileID.isInvalid() &&
589  "PCHThroughHeaderFileID already set!");
590  PCHThroughHeaderFileID = FID;
591 }
592 
594  assert(PCHThroughHeaderFileID.isValid() &&
595  "Invalid PCH through header FileID");
596  return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
597 }
598 
600  return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
601  PCHThroughHeaderFileID.isValid();
602 }
603 
605  return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
606  PCHThroughHeaderFileID.isValid();
607 }
608 
610  return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
611 }
612 
614  return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
615 }
616 
617 /// Skip tokens until after the #include of the through header or
618 /// until after a #pragma hdrstop is seen. Tokens in the predefines file
619 /// and the main file may be skipped. If the end of the predefines file
620 /// is reached, skipping continues into the main file. If the end of the
621 /// main file is reached, it's a fatal error.
623  bool ReachedMainFileEOF = false;
624  bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
625  bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
626  Token Tok;
627  while (true) {
628  bool InPredefines = (CurLexer->getFileID() == getPredefinesFileID());
629  CurLexer->Lex(Tok);
630  if (Tok.is(tok::eof) && !InPredefines) {
631  ReachedMainFileEOF = true;
632  break;
633  }
634  if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
635  break;
636  if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
637  break;
638  }
639  if (ReachedMainFileEOF) {
640  if (UsingPCHThroughHeader)
641  Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
642  << PPOpts->PCHThroughHeader << 1;
643  else if (!PPOpts->PCHWithHdrStopCreate)
644  Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
645  }
646 }
647 
648 void Preprocessor::replayPreambleConditionalStack() {
649  // Restore the conditional stack from the preamble, if there is one.
650  if (PreambleConditionalStack.isReplaying()) {
651  assert(CurPPLexer &&
652  "CurPPLexer is null when calling replayPreambleConditionalStack.");
653  CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
654  PreambleConditionalStack.doneReplaying();
655  if (PreambleConditionalStack.reachedEOFWhileSkipping())
656  SkipExcludedConditionalBlock(
657  PreambleConditionalStack.SkipInfo->HashTokenLoc,
658  PreambleConditionalStack.SkipInfo->IfTokenLoc,
659  PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
660  PreambleConditionalStack.SkipInfo->FoundElse,
661  PreambleConditionalStack.SkipInfo->ElseLoc);
662  }
663 }
664 
666  // Notify the client that we reached the end of the source file.
667  if (Callbacks)
668  Callbacks->EndOfMainFile();
669 }
670 
671 //===----------------------------------------------------------------------===//
672 // Lexer Event Handling.
673 //===----------------------------------------------------------------------===//
674 
675 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
676 /// identifier information for the token and install it into the token,
677 /// updating the token kind accordingly.
679  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
680 
681  // Look up this token, see if it is a macro, or if it is a language keyword.
682  IdentifierInfo *II;
683  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
684  // No cleaning needed, just use the characters from the lexed buffer.
685  II = getIdentifierInfo(Identifier.getRawIdentifier());
686  } else {
687  // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
688  SmallString<64> IdentifierBuffer;
689  StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
690 
691  if (Identifier.hasUCN()) {
692  SmallString<64> UCNIdentifierBuffer;
693  expandUCNs(UCNIdentifierBuffer, CleanedStr);
694  II = getIdentifierInfo(UCNIdentifierBuffer);
695  } else {
696  II = getIdentifierInfo(CleanedStr);
697  }
698  }
699 
700  // Update the token info (identifier info and appropriate token kind).
701  Identifier.setIdentifierInfo(II);
702  if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
704  Identifier.setKind(tok::identifier);
705  else
706  Identifier.setKind(II->getTokenID());
707 
708  return II;
709 }
710 
711 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
712  PoisonReasons[II] = DiagID;
713 }
714 
716  assert(Ident__exception_code && Ident__exception_info);
717  assert(Ident___exception_code && Ident___exception_info);
718  Ident__exception_code->setIsPoisoned(Poison);
719  Ident___exception_code->setIsPoisoned(Poison);
720  Ident_GetExceptionCode->setIsPoisoned(Poison);
721  Ident__exception_info->setIsPoisoned(Poison);
722  Ident___exception_info->setIsPoisoned(Poison);
723  Ident_GetExceptionInfo->setIsPoisoned(Poison);
724  Ident__abnormal_termination->setIsPoisoned(Poison);
725  Ident___abnormal_termination->setIsPoisoned(Poison);
726  Ident_AbnormalTermination->setIsPoisoned(Poison);
727 }
728 
730  assert(Identifier.getIdentifierInfo() &&
731  "Can't handle identifiers without identifier info!");
732  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
733  PoisonReasons.find(Identifier.getIdentifierInfo());
734  if(it == PoisonReasons.end())
735  Diag(Identifier, diag::err_pp_used_poisoned_id);
736  else
737  Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
738 }
739 
740 /// Returns a diagnostic message kind for reporting a future keyword as
741 /// appropriate for the identifier and specified language.
743  const LangOptions &LangOpts) {
744  assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
745 
746  if (LangOpts.CPlusPlus)
747  return llvm::StringSwitch<diag::kind>(II.getName())
748 #define CXX11_KEYWORD(NAME, FLAGS) \
749  .Case(#NAME, diag::warn_cxx11_keyword)
750 #define CXX2A_KEYWORD(NAME, FLAGS) \
751  .Case(#NAME, diag::warn_cxx2a_keyword)
752 #include "clang/Basic/TokenKinds.def"
753  ;
754 
755  llvm_unreachable(
756  "Keyword not known to come from a newer Standard or proposed Standard");
757 }
758 
759 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
760  assert(II.isOutOfDate() && "not out of date");
762 }
763 
764 /// HandleIdentifier - This callback is invoked when the lexer reads an
765 /// identifier. This callback looks up the identifier in the map and/or
766 /// potentially macro expands it or turns it into a named token (like 'for').
767 ///
768 /// Note that callers of this method are guarded by checking the
769 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
770 /// IdentifierInfo methods that compute these properties will need to change to
771 /// match.
773  assert(Identifier.getIdentifierInfo() &&
774  "Can't handle identifiers without identifier info!");
775 
776  IdentifierInfo &II = *Identifier.getIdentifierInfo();
777 
778  // If the information about this identifier is out of date, update it from
779  // the external source.
780  // We have to treat __VA_ARGS__ in a special way, since it gets
781  // serialized with isPoisoned = true, but our preprocessor may have
782  // unpoisoned it if we're defining a C99 macro.
783  if (II.isOutOfDate()) {
784  bool CurrentIsPoisoned = false;
785  const bool IsSpecialVariadicMacro =
786  &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
787  if (IsSpecialVariadicMacro)
788  CurrentIsPoisoned = II.isPoisoned();
789 
790  updateOutOfDateIdentifier(II);
791  Identifier.setKind(II.getTokenID());
792 
793  if (IsSpecialVariadicMacro)
794  II.setIsPoisoned(CurrentIsPoisoned);
795  }
796 
797  // If this identifier was poisoned, and if it was not produced from a macro
798  // expansion, emit an error.
799  if (II.isPoisoned() && CurPPLexer) {
800  HandlePoisonedIdentifier(Identifier);
801  }
802 
803  // If this is a macro to be expanded, do it.
804  if (MacroDefinition MD = getMacroDefinition(&II)) {
805  auto *MI = MD.getMacroInfo();
806  assert(MI && "macro definition with no macro info?");
807  if (!DisableMacroExpansion) {
808  if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
809  // C99 6.10.3p10: If the preprocessing token immediately after the
810  // macro name isn't a '(', this macro should not be expanded.
811  if (!MI->isFunctionLike() || isNextPPTokenLParen())
812  return HandleMacroExpandedIdentifier(Identifier, MD);
813  } else {
814  // C99 6.10.3.4p2 says that a disabled macro may never again be
815  // expanded, even if it's in a context where it could be expanded in the
816  // future.
817  Identifier.setFlag(Token::DisableExpand);
818  if (MI->isObjectLike() || isNextPPTokenLParen())
819  Diag(Identifier, diag::pp_disabled_macro_expansion);
820  }
821  }
822  }
823 
824  // If this identifier is a keyword in a newer Standard or proposed Standard,
825  // produce a warning. Don't warn if we're not considering macro expansion,
826  // since this identifier might be the name of a macro.
827  // FIXME: This warning is disabled in cases where it shouldn't be, like
828  // "#define constexpr constexpr", "int constexpr;"
829  if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
830  Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
831  << II.getName();
832  // Don't diagnose this keyword again in this translation unit.
833  II.setIsFutureCompatKeyword(false);
834  }
835 
836  // If this is an extension token, diagnose its use.
837  // We avoid diagnosing tokens that originate from macro definitions.
838  // FIXME: This warning is disabled in cases where it shouldn't be,
839  // like "#define TY typeof", "TY(1) x".
840  if (II.isExtensionToken() && !DisableMacroExpansion)
841  Diag(Identifier, diag::ext_token_used);
842 
843  // If this is the 'import' contextual keyword following an '@', note
844  // that the next token indicates a module name.
845  //
846  // Note that we do not treat 'import' as a contextual
847  // keyword when we're in a caching lexer, because caching lexers only get
848  // used in contexts where import declarations are disallowed.
849  //
850  // Likewise if this is the C++ Modules TS import keyword.
851  if (((LastTokenWasAt && II.isModulesImport()) ||
852  Identifier.is(tok::kw_import)) &&
853  !InMacroArgs && !DisableMacroExpansion &&
854  (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
855  CurLexerKind != CLK_CachingLexer) {
856  ModuleImportLoc = Identifier.getLocation();
857  ModuleImportPath.clear();
858  ModuleImportExpectsIdentifier = true;
859  CurLexerKind = CLK_LexAfterModuleImport;
860  }
861  return true;
862 }
863 
865  // We loop here until a lex function returns a token; this avoids recursion.
866  bool ReturnedToken;
867  do {
868  switch (CurLexerKind) {
869  case CLK_Lexer:
870  ReturnedToken = CurLexer->Lex(Result);
871  break;
872  case CLK_TokenLexer:
873  ReturnedToken = CurTokenLexer->Lex(Result);
874  break;
875  case CLK_CachingLexer:
876  CachingLex(Result);
877  ReturnedToken = true;
878  break;
879  case CLK_LexAfterModuleImport:
880  LexAfterModuleImport(Result);
881  ReturnedToken = true;
882  break;
883  }
884  } while (!ReturnedToken);
885 
886  if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
887  // Remember the identifier before code completion token.
890  // Set IdenfitierInfo to null to avoid confusing code that handles both
891  // identifiers and completion tokens.
892  Result.setIdentifierInfo(nullptr);
893  }
894 
895  LastTokenWasAt = Result.is(tok::at);
896 }
897 
898 /// Lex a header-name token (including one formed from header-name-tokens if
899 /// \p AllowConcatenation is \c true).
900 ///
901 /// \param FilenameTok Filled in with the next token. On success, this will
902 /// be either an angle_header_name or a string_literal token. On
903 /// failure, it will be whatever other token was found instead.
904 /// \param AllowMacroExpansion If \c true, allow the header name to be formed
905 /// by macro expansion (concatenating tokens as necessary if the first
906 /// token is a '<').
907 /// \return \c true if we reached EOD or EOF while looking for a > token in
908 /// a concatenated header name and diagnosed it. \c false otherwise.
909 bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
910  // Lex using header-name tokenization rules if tokens are being lexed from
911  // a file. Just grab a token normally if we're in a macro expansion.
912  if (CurPPLexer)
913  CurPPLexer->LexIncludeFilename(FilenameTok);
914  else
915  Lex(FilenameTok);
916 
917  // This could be a <foo/bar.h> file coming from a macro expansion. In this
918  // case, glue the tokens together into an angle_string_literal token.
919  SmallString<128> FilenameBuffer;
920  if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
921  SourceLocation Start = FilenameTok.getLocation();
923  FilenameBuffer.push_back('<');
924 
925  // Consume tokens until we find a '>'.
926  // FIXME: A header-name could be formed starting or ending with an
927  // alternative token. It's not clear whether that's ill-formed in all
928  // cases.
929  while (FilenameTok.isNot(tok::greater)) {
930  Lex(FilenameTok);
931  if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
932  Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
933  Diag(Start, diag::note_matching) << tok::less;
934  return true;
935  }
936 
937  End = FilenameTok.getLocation();
938 
939  // FIXME: Provide code completion for #includes.
940  if (FilenameTok.is(tok::code_completion)) {
942  Lex(FilenameTok);
943  continue;
944  }
945 
946  // Append the spelling of this token to the buffer. If there was a space
947  // before it, add it now.
948  if (FilenameTok.hasLeadingSpace())
949  FilenameBuffer.push_back(' ');
950 
951  // Get the spelling of the token, directly into FilenameBuffer if
952  // possible.
953  size_t PreAppendSize = FilenameBuffer.size();
954  FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
955 
956  const char *BufPtr = &FilenameBuffer[PreAppendSize];
957  unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
958 
959  // If the token was spelled somewhere else, copy it into FilenameBuffer.
960  if (BufPtr != &FilenameBuffer[PreAppendSize])
961  memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
962 
963  // Resize FilenameBuffer to the correct size.
964  if (FilenameTok.getLength() != ActualLen)
965  FilenameBuffer.resize(PreAppendSize + ActualLen);
966  }
967 
968  FilenameTok.startToken();
969  FilenameTok.setKind(tok::header_name);
970  CreateString(FilenameBuffer, FilenameTok, Start, End);
971  } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
972  // Convert a string-literal token of the form " h-char-sequence "
973  // (produced by macro expansion) into a header-name token.
974  //
975  // The rules for header-names don't quite match the rules for
976  // string-literals, but all the places where they differ result in
977  // undefined behavior, so we can and do treat them the same.
978  //
979  // A string-literal with a prefix or suffix is not translated into a
980  // header-name. This could theoretically be observable via the C++20
981  // context-sensitive header-name formation rules.
982  StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
983  if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
984  FilenameTok.setKind(tok::header_name);
985  }
986 
987  return false;
988 }
989 
990 /// Lex a token following the 'import' contextual keyword.
991 ///
993  // Figure out what kind of lexer we actually have.
995 
996  // Lex the next token.
997  Lex(Result);
998 
999  // The token sequence
1000  //
1001  // import identifier (. identifier)*
1002  //
1003  // indicates a module import directive. We already saw the 'import'
1004  // contextual keyword, so now we're looking for the identifiers.
1005  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1006  // We expected to see an identifier here, and we did; continue handling
1007  // identifiers.
1008  ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
1009  Result.getLocation()));
1010  ModuleImportExpectsIdentifier = false;
1011  CurLexerKind = CLK_LexAfterModuleImport;
1012  return;
1013  }
1014 
1015  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1016  // see the next identifier. (We can also see a '[[' that begins an
1017  // attribute-specifier-seq here under the C++ Modules TS.)
1018  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1019  ModuleImportExpectsIdentifier = true;
1020  CurLexerKind = CLK_LexAfterModuleImport;
1021  return;
1022  }
1023 
1024  // If we have a non-empty module path, load the named module.
1025  if (!ModuleImportPath.empty()) {
1026  // Under the Modules TS, the dot is just part of the module name, and not
1027  // a real hierarchy separator. Flatten such module names now.
1028  //
1029  // FIXME: Is this the right level to be performing this transformation?
1030  std::string FlatModuleName;
1031  if (getLangOpts().ModulesTS) {
1032  for (auto &Piece : ModuleImportPath) {
1033  if (!FlatModuleName.empty())
1034  FlatModuleName += ".";
1035  FlatModuleName += Piece.first->getName();
1036  }
1037  SourceLocation FirstPathLoc = ModuleImportPath[0].second;
1038  ModuleImportPath.clear();
1039  ModuleImportPath.push_back(
1040  std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1041  }
1042 
1043  Module *Imported = nullptr;
1044  if (getLangOpts().Modules) {
1045  Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1046  ModuleImportPath,
1048  /*IsIncludeDirective=*/false);
1049  if (Imported)
1050  makeModuleVisible(Imported, ModuleImportLoc);
1051  }
1052  if (Callbacks && (getLangOpts().Modules || getLangOpts().DebuggerSupport))
1053  Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1054  }
1055 }
1056 
1058  CurSubmoduleState->VisibleModules.setVisible(
1059  M, Loc, [](Module *) {},
1060  [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1061  // FIXME: Include the path in the diagnostic.
1062  // FIXME: Include the import location for the conflicting module.
1063  Diag(ModuleImportLoc, diag::warn_module_conflict)
1064  << Path[0]->getFullModuleName()
1065  << Conflict->getFullModuleName()
1066  << Message;
1067  });
1068 
1069  // Add this module to the imports list of the currently-built submodule.
1070  if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1071  BuildingSubmoduleStack.back().M->Imports.insert(M);
1072 }
1073 
1075  const char *DiagnosticTag,
1076  bool AllowMacroExpansion) {
1077  // We need at least one string literal.
1078  if (Result.isNot(tok::string_literal)) {
1079  Diag(Result, diag::err_expected_string_literal)
1080  << /*Source='in...'*/0 << DiagnosticTag;
1081  return false;
1082  }
1083 
1084  // Lex string literal tokens, optionally with macro expansion.
1085  SmallVector<Token, 4> StrToks;
1086  do {
1087  StrToks.push_back(Result);
1088 
1089  if (Result.hasUDSuffix())
1090  Diag(Result, diag::err_invalid_string_udl);
1091 
1092  if (AllowMacroExpansion)
1093  Lex(Result);
1094  else
1095  LexUnexpandedToken(Result);
1096  } while (Result.is(tok::string_literal));
1097 
1098  // Concatenate and parse the strings.
1099  StringLiteralParser Literal(StrToks, *this);
1100  assert(Literal.isAscii() && "Didn't allow wide strings in");
1101 
1102  if (Literal.hadError)
1103  return false;
1104 
1105  if (Literal.Pascal) {
1106  Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1107  << /*Source='in...'*/0 << DiagnosticTag;
1108  return false;
1109  }
1110 
1111  String = Literal.GetString();
1112  return true;
1113 }
1114 
1116  assert(Tok.is(tok::numeric_constant));
1117  SmallString<8> IntegerBuffer;
1118  bool NumberInvalid = false;
1119  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1120  if (NumberInvalid)
1121  return false;
1122  NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
1123  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1124  return false;
1125  llvm::APInt APVal(64, 0);
1126  if (Literal.GetIntegerValue(APVal))
1127  return false;
1128  Lex(Tok);
1129  Value = APVal.getLimitedValue();
1130  return true;
1131 }
1132 
1134  assert(Handler && "NULL comment handler");
1135  assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
1136  CommentHandlers.end() && "Comment handler already registered");
1137  CommentHandlers.push_back(Handler);
1138 }
1139 
1141  std::vector<CommentHandler *>::iterator Pos =
1142  std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
1143  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1144  CommentHandlers.erase(Pos);
1145 }
1146 
1148  bool AnyPendingTokens = false;
1149  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1150  HEnd = CommentHandlers.end();
1151  H != HEnd; ++H) {
1152  if ((*H)->HandleComment(*this, Comment))
1153  AnyPendingTokens = true;
1154  }
1155  if (!AnyPendingTokens || getCommentRetentionState())
1156  return false;
1157  Lex(result);
1158  return true;
1159 }
1160 
1161 ModuleLoader::~ModuleLoader() = default;
1162 
1164 
1166 
1168  if (Record)
1169  return;
1170 
1171  Record = new PreprocessingRecord(getSourceManager());
1172  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1173 }
StringRef Identifier
Definition: Format.cpp:1674
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void FinalizeForModelFile()
Cleanup after model file parsing.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:94
Defines the SourceManager interface.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Defines the clang::Module class, which describes a module in the source code.
Module * getCurrentModule()
Retrieves the module that we&#39;re currently building, if any.
Defines the FileSystemStatCache interface.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:367
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void LexAfterModuleImport(Token &Result)
Lex a token following the &#39;import&#39; contextual keyword.
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:233
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:242
tok::TokenKind getKind() const
Definition: Token.h:89
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:112
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
void print(raw_ostream &OS, const SourceManager &SM) const
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:206
Definition: Format.h:2118
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void setKind(tok::TokenKind K)
Definition: Token.h:90
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive...
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:814
Describes a module or submodule.
Definition: Module.h:64
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
virtual ~ModuleLoader()
bool isInvalid() const
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:819
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:235
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
Definition: Token.h:290
const FormatToken & Tok
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:148
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
bool HandleComment(Token &result, SourceRange Comment)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
Provides lookups to, and iteration over, IdentiferInfo objects.
bool isValid() const
Exposes information about the current target.
Definition: TargetInfo.h:161
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
SourceLocation End
void makeModuleVisible(Module *M, SourceLocation Loc)
SourceLocation getLocation() const
Definition: MacroInfo.h:462
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:123
Defines the clang::Preprocessor interface.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
size_t getTotalMemory() const
bool getCommentRetentionState() const
Definition: Preprocessor.h:853
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isObjectLike() const
Definition: MacroInfo.h:200
void IncrementIncludeCount(const FileEntry *File)
Increment the count for the number of times the specified FileEntry has been entered.
Definition: HeaderSearch.h:459
The result type of a method or function.
StringRef GetString() const
const SourceManager & SM
Definition: Format.cpp:1528
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
Definition: Preprocessor.h:818
void DumpMacro(const MacroInfo &MI) const
void overrideFileContents(const FileEntry *SourceFile, llvm::MemoryBuffer *Buffer, bool DoNotFree)
Override the contents of the given source file by providing an already-allocated buffer.
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:831
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6...
Definition: Token.h:273
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:132
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:938
All of the names in this module are hidden.
Definition: Module.h:268
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:176
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:185
void Lex(Token &Result)
Lex the next token for this preprocessor.
llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:265
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Definition: Token.h:95
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:233
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
FileID getMainFileID() const
Returns the FileID of the main source file.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:96
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Abstract interface for a module loader.
Definition: ModuleLoader.h:73
unsigned getLength() const
Definition: Token.h:126
void PoisonSEHIdentifiers(bool Poison=true)
void setLiteralData(const char *Ptr)
Definition: Token.h:218
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:89
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token&#39;s location and length to ...
Module * lookupModule(StringRef ModuleName, bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
#define CXX11_KEYWORD(NAME, FLAGS)
Defines the clang::SourceLocation class and associated facilities.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:898
StringRef getRawIdentifier() const
getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode), returns a reference to the text substring in the buffer if known.
Definition: Token.h:202
SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc, SourceLocation TokenStart, SourceLocation TokenEnd)
Return a new SourceLocation that encodes that the token starting at TokenStart ends prematurely at To...
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:361
void DumpLocation(SourceLocation Loc) const
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Definition: DiagnosticIDs.h:60
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:24
Defines the clang::TargetInfo interface.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:282
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber...
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void setLocation(SourceLocation L)
Definition: Token.h:131
#define true
Definition: stdbool.h:32
A trivial tuple used to represent a source range.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:293
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Defines the PreprocessorLexer interface.
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:24
void InitializeTarget(const TargetInfo &Target, const TargetInfo *AuxTarget)
Perform target-specific initialization.
Definition: Builtins.cpp:42
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:269
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:906
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Definition: Token.h:168
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
SourceLocation getEndLoc() const
Definition: Token.h:150