clang 18.0.0git
Preprocessor.cpp
Go to the documentation of this file.
1//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Preprocessor interface.
10//
11//===----------------------------------------------------------------------===//
12//
13// Options to support:
14// -H - Print the name of each header file used.
15// -d[DNI] - Dump various things.
16// -fworking-directory - #line's with preprocessor's working dir.
17// -fpreprocessed
18// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19// -W*
20// -w
21//
22// Messages to emit:
23// "Multiple include guards may be useful for:\n"
24//
25//===----------------------------------------------------------------------===//
26
32#include "clang/Basic/LLVM.h"
34#include "clang/Basic/Module.h"
42#include "clang/Lex/Lexer.h"
44#include "clang/Lex/MacroArgs.h"
45#include "clang/Lex/MacroInfo.h"
47#include "clang/Lex/Pragma.h"
52#include "clang/Lex/Token.h"
54#include "llvm/ADT/APInt.h"
55#include "llvm/ADT/ArrayRef.h"
56#include "llvm/ADT/DenseMap.h"
57#include "llvm/ADT/STLExtras.h"
58#include "llvm/ADT/SmallString.h"
59#include "llvm/ADT/SmallVector.h"
60#include "llvm/ADT/StringRef.h"
61#include "llvm/Support/Capacity.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/MemoryBuffer.h"
64#include "llvm/Support/raw_ostream.h"
65#include <algorithm>
66#include <cassert>
67#include <memory>
68#include <optional>
69#include <string>
70#include <utility>
71#include <vector>
72
73using namespace clang;
74
75LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
76
78
79Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
80 DiagnosticsEngine &diags, const LangOptions &opts,
81 SourceManager &SM, HeaderSearch &Headers,
82 ModuleLoader &TheModuleLoader,
83 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
85 : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
87 ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
89 // As the language options may have not been loaded yet (when
90 // deserializing an ASTUnit), adding keywords to the identifier table is
91 // deferred to Preprocessor::Initialize().
92 Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
93 TUKind(TUKind), SkipMainFilePreamble(0, true),
94 CurSubmoduleState(&NullSubmoduleState) {
95 OwnsHeaderSearch = OwnsHeaders;
96
97 // Default to discarding comments.
98 KeepComments = false;
99 KeepMacroComments = false;
100 SuppressIncludeNotFoundError = false;
101
102 // Macro expansion is enabled.
103 DisableMacroExpansion = false;
104 MacroExpansionInDirectivesOverride = false;
105 InMacroArgs = false;
106 ArgMacro = nullptr;
107 InMacroArgPreExpansion = false;
108 NumCachedTokenLexers = 0;
109 PragmasEnabled = true;
110 ParsingIfOrElifDirective = false;
111 PreprocessedOutput = false;
112
113 // We haven't read anything from the external source.
114 ReadMacrosFromExternalSource = false;
115
116 BuiltinInfo = std::make_unique<Builtin::Context>();
117
118 // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
119 // a macro. They get unpoisoned where it is allowed.
120 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
121 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
122 (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
123 SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
124
125 // Initialize the pragma handlers.
126 RegisterBuiltinPragmas();
127
128 // Initialize builtin macros like __LINE__ and friends.
129 RegisterBuiltinMacros();
130
131 if(LangOpts.Borland) {
132 Ident__exception_info = getIdentifierInfo("_exception_info");
133 Ident___exception_info = getIdentifierInfo("__exception_info");
134 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
135 Ident__exception_code = getIdentifierInfo("_exception_code");
136 Ident___exception_code = getIdentifierInfo("__exception_code");
137 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
138 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
139 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
140 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
141 } else {
142 Ident__exception_info = Ident__exception_code = nullptr;
143 Ident__abnormal_termination = Ident___exception_info = nullptr;
144 Ident___exception_code = Ident___abnormal_termination = nullptr;
145 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
146 Ident_AbnormalTermination = nullptr;
147 }
148
149 // Default incremental processing to -fincremental-extensions, clients can
150 // override with `enableIncrementalProcessing` if desired.
151 IncrementalProcessing = LangOpts.IncrementalExtensions;
152
153 // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
155 SkippingUntilPragmaHdrStop = true;
156
157 // If using a PCH with a through header, start skipping tokens.
158 if (!this->PPOpts->PCHThroughHeader.empty() &&
159 !this->PPOpts->ImplicitPCHInclude.empty())
160 SkippingUntilPCHThroughHeader = true;
161
162 if (this->PPOpts->GeneratePreamble)
163 PreambleConditionalStack.startRecording();
164
165 MaxTokens = LangOpts.MaxTokens;
166}
167
169 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
170
171 IncludeMacroStack.clear();
172
173 // Free any cached macro expanders.
174 // This populates MacroArgCache, so all TokenLexers need to be destroyed
175 // before the code below that frees up the MacroArgCache list.
176 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
177 CurTokenLexer.reset();
178
179 // Free any cached MacroArgs.
180 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
181 ArgList = ArgList->deallocate();
182
183 // Delete the header search info, if we own it.
184 if (OwnsHeaderSearch)
185 delete &HeaderInfo;
186}
187
189 const TargetInfo *AuxTarget) {
190 assert((!this->Target || this->Target == &Target) &&
191 "Invalid override of target information");
192 this->Target = &Target;
193
194 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
195 "Invalid override of aux target information.");
196 this->AuxTarget = AuxTarget;
197
198 // Initialize information about built-ins.
199 BuiltinInfo->InitializeTarget(Target, AuxTarget);
200 HeaderInfo.setTarget(Target);
201
202 // Populate the identifier table with info about keywords for the current language.
203 Identifiers.AddKeywords(LangOpts);
204
205 // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
206 setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
207
208 if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine)
209 // Use setting from TargetInfo.
210 setCurrentFPEvalMethod(SourceLocation(), Target.getFPEvalMethod());
211 else
212 // Set initial value of __FLT_EVAL_METHOD__ from the command line.
213 setCurrentFPEvalMethod(SourceLocation(), getLangOpts().getFPEvalMethod());
214}
215
217 NumEnteredSourceFiles = 0;
218
219 // Reset pragmas
220 PragmaHandlersBackup = std::move(PragmaHandlers);
221 PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
222 RegisterBuiltinPragmas();
223
224 // Reset PredefinesFileID
225 PredefinesFileID = FileID();
226}
227
229 NumEnteredSourceFiles = 1;
230
231 PragmaHandlers = std::move(PragmaHandlersBackup);
232}
233
234void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
235 llvm::errs() << tok::getTokenName(Tok.getKind());
236
237 if (!Tok.isAnnotation())
238 llvm::errs() << " '" << getSpelling(Tok) << "'";
239
240 if (!DumpFlags) return;
241
242 llvm::errs() << "\t";
243 if (Tok.isAtStartOfLine())
244 llvm::errs() << " [StartOfLine]";
245 if (Tok.hasLeadingSpace())
246 llvm::errs() << " [LeadingSpace]";
247 if (Tok.isExpandDisabled())
248 llvm::errs() << " [ExpandDisabled]";
249 if (Tok.needsCleaning()) {
250 const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
251 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
252 << "']";
253 }
254
255 llvm::errs() << "\tLoc=<";
257 llvm::errs() << ">";
258}
259
261 Loc.print(llvm::errs(), SourceMgr);
262}
263
264void Preprocessor::DumpMacro(const MacroInfo &MI) const {
265 llvm::errs() << "MACRO: ";
266 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
268 llvm::errs() << " ";
269 }
270 llvm::errs() << "\n";
271}
272
274 llvm::errs() << "\n*** Preprocessor Stats:\n";
275 llvm::errs() << NumDirectives << " directives found:\n";
276 llvm::errs() << " " << NumDefined << " #define.\n";
277 llvm::errs() << " " << NumUndefined << " #undef.\n";
278 llvm::errs() << " #include/#include_next/#import:\n";
279 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
280 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
281 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
282 llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
283 llvm::errs() << " " << NumEndif << " #endif.\n";
284 llvm::errs() << " " << NumPragma << " #pragma.\n";
285 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
286
287 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
288 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
289 << NumFastMacroExpanded << " on the fast path.\n";
290 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
291 << " token paste (##) operations performed, "
292 << NumFastTokenPaste << " on the fast path.\n";
293
294 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
295
296 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
297 llvm::errs() << "\n Macro Expanded Tokens: "
298 << llvm::capacity_in_bytes(MacroExpandedTokens);
299 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
300 // FIXME: List information for all submodules.
301 llvm::errs() << "\n Macros: "
302 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
303 llvm::errs() << "\n #pragma push_macro Info: "
304 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
305 llvm::errs() << "\n Poison Reasons: "
306 << llvm::capacity_in_bytes(PoisonReasons);
307 llvm::errs() << "\n Comment Handlers: "
308 << llvm::capacity_in_bytes(CommentHandlers) << "\n";
309}
310
312Preprocessor::macro_begin(bool IncludeExternalMacros) const {
313 if (IncludeExternalMacros && ExternalSource &&
314 !ReadMacrosFromExternalSource) {
315 ReadMacrosFromExternalSource = true;
316 ExternalSource->ReadDefinedMacros();
317 }
318
319 // Make sure we cover all macros in visible modules.
320 for (const ModuleMacro &Macro : ModuleMacros)
321 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
322
323 return CurSubmoduleState->Macros.begin();
324}
325
327 return BP.getTotalMemory()
328 + llvm::capacity_in_bytes(MacroExpandedTokens)
329 + Predefines.capacity() /* Predefines buffer. */
330 // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
331 // and ModuleMacros.
332 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
333 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
334 + llvm::capacity_in_bytes(PoisonReasons)
335 + llvm::capacity_in_bytes(CommentHandlers);
336}
337
339Preprocessor::macro_end(bool IncludeExternalMacros) const {
340 if (IncludeExternalMacros && ExternalSource &&
341 !ReadMacrosFromExternalSource) {
342 ReadMacrosFromExternalSource = true;
343 ExternalSource->ReadDefinedMacros();
344 }
345
346 return CurSubmoduleState->Macros.end();
347}
348
349/// Compares macro tokens with a specified token value sequence.
350static bool MacroDefinitionEquals(const MacroInfo *MI,
351 ArrayRef<TokenValue> Tokens) {
352 return Tokens.size() == MI->getNumTokens() &&
353 std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
354}
355
357 SourceLocation Loc,
358 ArrayRef<TokenValue> Tokens) const {
359 SourceLocation BestLocation;
360 StringRef BestSpelling;
362 I != E; ++I) {
364 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
365 if (!Def || !Def.getMacroInfo())
366 continue;
367 if (!Def.getMacroInfo()->isObjectLike())
368 continue;
369 if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
370 continue;
371 SourceLocation Location = Def.getLocation();
372 // Choose the macro defined latest.
373 if (BestLocation.isInvalid() ||
374 (Location.isValid() &&
375 SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
376 BestLocation = Location;
377 BestSpelling = I->first->getName();
378 }
379 }
380 return BestSpelling;
381}
382
384 if (CurLexer)
385 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
386 ? CLK_DependencyDirectivesLexer
387 : CLK_Lexer;
388 else if (CurTokenLexer)
389 CurLexerCallback = CLK_TokenLexer;
390 else
391 CurLexerCallback = CLK_CachingLexer;
392}
393
395 unsigned CompleteLine,
396 unsigned CompleteColumn) {
397 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
398 assert(!CodeCompletionFile && "Already set");
399
400 // Load the actual file's contents.
401 std::optional<llvm::MemoryBufferRef> Buffer =
403 if (!Buffer)
404 return true;
405
406 // Find the byte position of the truncation point.
407 const char *Position = Buffer->getBufferStart();
408 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
409 for (; *Position; ++Position) {
410 if (*Position != '\r' && *Position != '\n')
411 continue;
412
413 // Eat \r\n or \n\r as a single line.
414 if ((Position[1] == '\r' || Position[1] == '\n') &&
415 Position[0] != Position[1])
416 ++Position;
417 ++Position;
418 break;
419 }
420 }
421
422 Position += CompleteColumn - 1;
423
424 // If pointing inside the preamble, adjust the position at the beginning of
425 // the file after the preamble.
426 if (SkipMainFilePreamble.first &&
427 SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
428 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
429 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
430 }
431
432 if (Position > Buffer->getBufferEnd())
433 Position = Buffer->getBufferEnd();
434
435 CodeCompletionFile = File;
436 CodeCompletionOffset = Position - Buffer->getBufferStart();
437
438 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
439 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
440 char *NewBuf = NewBuffer->getBufferStart();
441 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
442 *NewPos = '\0';
443 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
444 SourceMgr.overrideFileContents(File, std::move(NewBuffer));
445
446 return false;
447}
448
450 bool IsAngled) {
452 if (CodeComplete)
453 CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
454}
455
458 if (CodeComplete)
459 CodeComplete->CodeCompleteNaturalLanguage();
460}
461
462/// getSpelling - This method is used to get the spelling of a token into a
463/// SmallVector. Note that the returned StringRef may not point to the
464/// supplied buffer if a copy can be avoided.
465StringRef Preprocessor::getSpelling(const Token &Tok,
466 SmallVectorImpl<char> &Buffer,
467 bool *Invalid) const {
468 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
469 if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
470 // Try the fast path.
471 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
472 return II->getName();
473 }
474
475 // Resize the buffer if we need to copy into it.
476 if (Tok.needsCleaning())
477 Buffer.resize(Tok.getLength());
478
479 const char *Ptr = Buffer.data();
480 unsigned Len = getSpelling(Tok, Ptr, Invalid);
481 return StringRef(Ptr, Len);
482}
483
484/// CreateString - Plop the specified string into a scratch buffer and return a
485/// location for it. If specified, the source location provides a source
486/// location for the token.
487void Preprocessor::CreateString(StringRef Str, Token &Tok,
488 SourceLocation ExpansionLocStart,
489 SourceLocation ExpansionLocEnd) {
490 Tok.setLength(Str.size());
491
492 const char *DestPtr;
493 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
494
495 if (ExpansionLocStart.isValid())
496 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
497 ExpansionLocEnd, Str.size());
498 Tok.setLocation(Loc);
499
500 // If this is a raw identifier or a literal token, set the pointer data.
501 if (Tok.is(tok::raw_identifier))
502 Tok.setRawIdentifierData(DestPtr);
503 else if (Tok.isLiteral())
504 Tok.setLiteralData(DestPtr);
505}
506
508 auto &SM = getSourceManager();
509 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
510 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
511 bool Invalid = false;
512 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
513 if (Invalid)
514 return SourceLocation();
515
516 // FIXME: We could consider re-using spelling for tokens we see repeatedly.
517 const char *DestPtr;
518 SourceLocation Spelling =
519 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
520 return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
521}
522
524 if (!getLangOpts().isCompilingModule())
525 return nullptr;
526
527 return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
528}
529
531 if (!getLangOpts().isCompilingModuleImplementation())
532 return nullptr;
533
534 return getHeaderSearchInfo().lookupModule(getLangOpts().ModuleName);
535}
536
537//===----------------------------------------------------------------------===//
538// Preprocessor Initialization Methods
539//===----------------------------------------------------------------------===//
540
541/// EnterMainSourceFile - Enter the specified FileID as the main source file,
542/// which implicitly adds the builtin defines etc.
544 // We do not allow the preprocessor to reenter the main file. Doing so will
545 // cause FileID's to accumulate information from both runs (e.g. #line
546 // information) and predefined macros aren't guaranteed to be set properly.
547 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
548 FileID MainFileID = SourceMgr.getMainFileID();
549
550 // If MainFileID is loaded it means we loaded an AST file, no need to enter
551 // a main file.
552 if (!SourceMgr.isLoadedFileID(MainFileID)) {
553 // Enter the main file source buffer.
554 EnterSourceFile(MainFileID, nullptr, SourceLocation());
555
556 // If we've been asked to skip bytes in the main file (e.g., as part of a
557 // precompiled preamble), do so now.
558 if (SkipMainFilePreamble.first > 0)
559 CurLexer->SetByteOffset(SkipMainFilePreamble.first,
560 SkipMainFilePreamble.second);
561
562 // Tell the header info that the main file was entered. If the file is later
563 // #imported, it won't be re-entered.
564 if (OptionalFileEntryRef FE = SourceMgr.getFileEntryRefForID(MainFileID))
565 markIncluded(*FE);
566 }
567
568 // Preprocess Predefines to populate the initial preprocessor state.
569 std::unique_ptr<llvm::MemoryBuffer> SB =
570 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
571 assert(SB && "Cannot create predefined source buffer");
572 FileID FID = SourceMgr.createFileID(std::move(SB));
573 assert(FID.isValid() && "Could not create FileID for predefines?");
574 setPredefinesFileID(FID);
575
576 // Start parsing the predefines.
577 EnterSourceFile(FID, nullptr, SourceLocation());
578
579 if (!PPOpts->PCHThroughHeader.empty()) {
580 // Lookup and save the FileID for the through header. If it isn't found
581 // in the search path, it's a fatal error.
583 SourceLocation(), PPOpts->PCHThroughHeader,
584 /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
585 /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
586 /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
587 /*IsFrameworkFound=*/nullptr);
588 if (!File) {
589 Diag(SourceLocation(), diag::err_pp_through_header_not_found)
590 << PPOpts->PCHThroughHeader;
591 return;
592 }
593 setPCHThroughHeaderFileID(
595 }
596
597 // Skip tokens from the Predefines and if needed the main file.
598 if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
599 (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
601}
602
603void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
604 assert(PCHThroughHeaderFileID.isInvalid() &&
605 "PCHThroughHeaderFileID already set!");
606 PCHThroughHeaderFileID = FID;
607}
608
610 assert(PCHThroughHeaderFileID.isValid() &&
611 "Invalid PCH through header FileID");
612 return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
613}
614
616 return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
617 PCHThroughHeaderFileID.isValid();
618}
619
621 return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
622 PCHThroughHeaderFileID.isValid();
623}
624
626 return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
627}
628
630 return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
631}
632
633/// Skip tokens until after the #include of the through header or
634/// until after a #pragma hdrstop is seen. Tokens in the predefines file
635/// and the main file may be skipped. If the end of the predefines file
636/// is reached, skipping continues into the main file. If the end of the
637/// main file is reached, it's a fatal error.
639 bool ReachedMainFileEOF = false;
640 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
641 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
642 Token Tok;
643 while (true) {
644 bool InPredefines =
645 (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
646 CurLexerCallback(*this, Tok);
647 if (Tok.is(tok::eof) && !InPredefines) {
648 ReachedMainFileEOF = true;
649 break;
650 }
651 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
652 break;
653 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
654 break;
655 }
656 if (ReachedMainFileEOF) {
657 if (UsingPCHThroughHeader)
658 Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
659 << PPOpts->PCHThroughHeader << 1;
660 else if (!PPOpts->PCHWithHdrStopCreate)
661 Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
662 }
663}
664
665void Preprocessor::replayPreambleConditionalStack() {
666 // Restore the conditional stack from the preamble, if there is one.
667 if (PreambleConditionalStack.isReplaying()) {
668 assert(CurPPLexer &&
669 "CurPPLexer is null when calling replayPreambleConditionalStack.");
670 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
671 PreambleConditionalStack.doneReplaying();
672 if (PreambleConditionalStack.reachedEOFWhileSkipping())
673 SkipExcludedConditionalBlock(
674 PreambleConditionalStack.SkipInfo->HashTokenLoc,
675 PreambleConditionalStack.SkipInfo->IfTokenLoc,
676 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
677 PreambleConditionalStack.SkipInfo->FoundElse,
678 PreambleConditionalStack.SkipInfo->ElseLoc);
679 }
680}
681
683 // Notify the client that we reached the end of the source file.
684 if (Callbacks)
685 Callbacks->EndOfMainFile();
686}
687
688//===----------------------------------------------------------------------===//
689// Lexer Event Handling.
690//===----------------------------------------------------------------------===//
691
692/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
693/// identifier information for the token and install it into the token,
694/// updating the token kind accordingly.
696 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
697
698 // Look up this token, see if it is a macro, or if it is a language keyword.
699 IdentifierInfo *II;
700 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
701 // No cleaning needed, just use the characters from the lexed buffer.
702 II = getIdentifierInfo(Identifier.getRawIdentifier());
703 } else {
704 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
705 SmallString<64> IdentifierBuffer;
706 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
707
708 if (Identifier.hasUCN()) {
709 SmallString<64> UCNIdentifierBuffer;
710 expandUCNs(UCNIdentifierBuffer, CleanedStr);
711 II = getIdentifierInfo(UCNIdentifierBuffer);
712 } else {
713 II = getIdentifierInfo(CleanedStr);
714 }
715 }
716
717 // Update the token info (identifier info and appropriate token kind).
718 // FIXME: the raw_identifier may contain leading whitespace which is removed
719 // from the cleaned identifier token. The SourceLocation should be updated to
720 // refer to the non-whitespace character. For instance, the text "\\\nB" (a
721 // line continuation before 'B') is parsed as a single tok::raw_identifier and
722 // is cleaned to tok::identifier "B". After cleaning the token's length is
723 // still 3 and the SourceLocation refers to the location of the backslash.
724 Identifier.setIdentifierInfo(II);
725 Identifier.setKind(II->getTokenID());
726
727 return II;
728}
729
731 PoisonReasons[II] = DiagID;
732}
733
735 assert(Ident__exception_code && Ident__exception_info);
736 assert(Ident___exception_code && Ident___exception_info);
737 Ident__exception_code->setIsPoisoned(Poison);
738 Ident___exception_code->setIsPoisoned(Poison);
739 Ident_GetExceptionCode->setIsPoisoned(Poison);
740 Ident__exception_info->setIsPoisoned(Poison);
741 Ident___exception_info->setIsPoisoned(Poison);
742 Ident_GetExceptionInfo->setIsPoisoned(Poison);
743 Ident__abnormal_termination->setIsPoisoned(Poison);
744 Ident___abnormal_termination->setIsPoisoned(Poison);
745 Ident_AbnormalTermination->setIsPoisoned(Poison);
746}
747
749 assert(Identifier.getIdentifierInfo() &&
750 "Can't handle identifiers without identifier info!");
751 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
752 PoisonReasons.find(Identifier.getIdentifierInfo());
753 if(it == PoisonReasons.end())
754 Diag(Identifier, diag::err_pp_used_poisoned_id);
755 else
756 Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
757}
758
759void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
760 assert(II.isOutOfDate() && "not out of date");
762}
763
764/// HandleIdentifier - This callback is invoked when the lexer reads an
765/// identifier. This callback looks up the identifier in the map and/or
766/// potentially macro expands it or turns it into a named token (like 'for').
767///
768/// Note that callers of this method are guarded by checking the
769/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
770/// IdentifierInfo methods that compute these properties will need to change to
771/// match.
773 assert(Identifier.getIdentifierInfo() &&
774 "Can't handle identifiers without identifier info!");
775
776 IdentifierInfo &II = *Identifier.getIdentifierInfo();
777
778 // If the information about this identifier is out of date, update it from
779 // the external source.
780 // We have to treat __VA_ARGS__ in a special way, since it gets
781 // serialized with isPoisoned = true, but our preprocessor may have
782 // unpoisoned it if we're defining a C99 macro.
783 if (II.isOutOfDate()) {
784 bool CurrentIsPoisoned = false;
785 const bool IsSpecialVariadicMacro =
786 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
787 if (IsSpecialVariadicMacro)
788 CurrentIsPoisoned = II.isPoisoned();
789
790 updateOutOfDateIdentifier(II);
791 Identifier.setKind(II.getTokenID());
792
793 if (IsSpecialVariadicMacro)
794 II.setIsPoisoned(CurrentIsPoisoned);
795 }
796
797 // If this identifier was poisoned, and if it was not produced from a macro
798 // expansion, emit an error.
799 if (II.isPoisoned() && CurPPLexer) {
801 }
802
803 // If this is a macro to be expanded, do it.
804 if (const MacroDefinition MD = getMacroDefinition(&II)) {
805 const auto *MI = MD.getMacroInfo();
806 assert(MI && "macro definition with no macro info?");
807 if (!DisableMacroExpansion) {
808 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
809 // C99 6.10.3p10: If the preprocessing token immediately after the
810 // macro name isn't a '(', this macro should not be expanded.
811 if (!MI->isFunctionLike() || isNextPPTokenLParen())
812 return HandleMacroExpandedIdentifier(Identifier, MD);
813 } else {
814 // C99 6.10.3.4p2 says that a disabled macro may never again be
815 // expanded, even if it's in a context where it could be expanded in the
816 // future.
818 if (MI->isObjectLike() || isNextPPTokenLParen())
819 Diag(Identifier, diag::pp_disabled_macro_expansion);
820 }
821 }
822 }
823
824 // If this identifier is a keyword in a newer Standard or proposed Standard,
825 // produce a warning. Don't warn if we're not considering macro expansion,
826 // since this identifier might be the name of a macro.
827 // FIXME: This warning is disabled in cases where it shouldn't be, like
828 // "#define constexpr constexpr", "int constexpr;"
829 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
830 Diag(Identifier, getIdentifierTable().getFutureCompatDiagKind(II, getLangOpts()))
831 << II.getName();
832 // Don't diagnose this keyword again in this translation unit.
833 II.setIsFutureCompatKeyword(false);
834 }
835
836 // If this is an extension token, diagnose its use.
837 // We avoid diagnosing tokens that originate from macro definitions.
838 // FIXME: This warning is disabled in cases where it shouldn't be,
839 // like "#define TY typeof", "TY(1) x".
840 if (II.isExtensionToken() && !DisableMacroExpansion)
841 Diag(Identifier, diag::ext_token_used);
842
843 // If this is the 'import' contextual keyword following an '@', note
844 // that the next token indicates a module name.
845 //
846 // Note that we do not treat 'import' as a contextual
847 // keyword when we're in a caching lexer, because caching lexers only get
848 // used in contexts where import declarations are disallowed.
849 //
850 // Likewise if this is the standard C++ import keyword.
851 if (((LastTokenWasAt && II.isModulesImport()) ||
852 Identifier.is(tok::kw_import)) &&
853 !InMacroArgs && !DisableMacroExpansion &&
854 (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
855 CurLexerCallback != CLK_CachingLexer) {
856 ModuleImportLoc = Identifier.getLocation();
857 NamedModuleImportPath.clear();
858 IsAtImport = true;
859 ModuleImportExpectsIdentifier = true;
860 CurLexerCallback = CLK_LexAfterModuleImport;
861 }
862 return true;
863}
864
866 ++LexLevel;
867
868 // We loop here until a lex function returns a token; this avoids recursion.
869 while (!CurLexerCallback(*this, Result))
870 ;
871
872 if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure)
873 return;
874
875 if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
876 // Remember the identifier before code completion token.
877 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
878 setCodeCompletionTokenRange(Result.getLocation(), Result.getEndLoc());
879 // Set IdenfitierInfo to null to avoid confusing code that handles both
880 // identifiers and completion tokens.
881 Result.setIdentifierInfo(nullptr);
882 }
883
884 // Update StdCXXImportSeqState to track our position within a C++20 import-seq
885 // if this token is being produced as a result of phase 4 of translation.
886 // Update TrackGMFState to decide if we are currently in a Global Module
887 // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state
888 // depends on the prevailing StdCXXImportSeq state in two cases.
889 if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
890 !Result.getFlag(Token::IsReinjected)) {
891 switch (Result.getKind()) {
892 case tok::l_paren: case tok::l_square: case tok::l_brace:
893 StdCXXImportSeqState.handleOpenBracket();
894 break;
895 case tok::r_paren: case tok::r_square:
896 StdCXXImportSeqState.handleCloseBracket();
897 break;
898 case tok::r_brace:
899 StdCXXImportSeqState.handleCloseBrace();
900 break;
901 // This token is injected to represent the translation of '#include "a.h"'
902 // into "import a.h;". Mimic the notional ';'.
903 case tok::annot_module_include:
904 case tok::semi:
905 TrackGMFState.handleSemi();
906 StdCXXImportSeqState.handleSemi();
907 ModuleDeclState.handleSemi();
908 break;
909 case tok::header_name:
910 case tok::annot_header_unit:
911 StdCXXImportSeqState.handleHeaderName();
912 break;
913 case tok::kw_export:
914 TrackGMFState.handleExport();
915 StdCXXImportSeqState.handleExport();
916 ModuleDeclState.handleExport();
917 break;
918 case tok::colon:
919 ModuleDeclState.handleColon();
920 break;
921 case tok::period:
922 ModuleDeclState.handlePeriod();
923 break;
924 case tok::identifier:
925 // Check "import" and "module" when there is no open bracket. The two
926 // identifiers are not meaningful with open brackets.
927 if (StdCXXImportSeqState.atTopLevel()) {
928 if (Result.getIdentifierInfo()->isModulesImport()) {
929 TrackGMFState.handleImport(StdCXXImportSeqState.afterTopLevelSeq());
930 StdCXXImportSeqState.handleImport();
931 if (StdCXXImportSeqState.afterImportSeq()) {
932 ModuleImportLoc = Result.getLocation();
933 NamedModuleImportPath.clear();
934 IsAtImport = false;
935 ModuleImportExpectsIdentifier = true;
936 CurLexerCallback = CLK_LexAfterModuleImport;
937 }
938 break;
939 } else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) {
940 TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
941 ModuleDeclState.handleModule();
942 break;
943 }
944 }
945 ModuleDeclState.handleIdentifier(Result.getIdentifierInfo());
946 if (ModuleDeclState.isModuleCandidate())
947 break;
948 [[fallthrough]];
949 default:
950 TrackGMFState.handleMisc();
951 StdCXXImportSeqState.handleMisc();
952 ModuleDeclState.handleMisc();
953 break;
954 }
955 }
956
957 LastTokenWasAt = Result.is(tok::at);
958 --LexLevel;
959
960 if ((LexLevel == 0 || PreprocessToken) &&
961 !Result.getFlag(Token::IsReinjected)) {
962 if (LexLevel == 0)
963 ++TokenCount;
964 if (OnToken)
965 OnToken(Result);
966 }
967}
968
969void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
970 while (1) {
971 Token Tok;
972 Lex(Tok);
973 if (Tok.isOneOf(tok::unknown, tok::eof, tok::eod,
974 tok::annot_repl_input_end))
975 break;
976 if (Tokens != nullptr)
977 Tokens->push_back(Tok);
978 }
979}
980
981/// Lex a header-name token (including one formed from header-name-tokens if
982/// \p AllowConcatenation is \c true).
983///
984/// \param FilenameTok Filled in with the next token. On success, this will
985/// be either a header_name token. On failure, it will be whatever other
986/// token was found instead.
987/// \param AllowMacroExpansion If \c true, allow the header name to be formed
988/// by macro expansion (concatenating tokens as necessary if the first
989/// token is a '<').
990/// \return \c true if we reached EOD or EOF while looking for a > token in
991/// a concatenated header name and diagnosed it. \c false otherwise.
992bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
993 // Lex using header-name tokenization rules if tokens are being lexed from
994 // a file. Just grab a token normally if we're in a macro expansion.
995 if (CurPPLexer)
996 CurPPLexer->LexIncludeFilename(FilenameTok);
997 else
998 Lex(FilenameTok);
999
1000 // This could be a <foo/bar.h> file coming from a macro expansion. In this
1001 // case, glue the tokens together into an angle_string_literal token.
1002 SmallString<128> FilenameBuffer;
1003 if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
1004 bool StartOfLine = FilenameTok.isAtStartOfLine();
1005 bool LeadingSpace = FilenameTok.hasLeadingSpace();
1006 bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1007
1008 SourceLocation Start = FilenameTok.getLocation();
1009 SourceLocation End;
1010 FilenameBuffer.push_back('<');
1011
1012 // Consume tokens until we find a '>'.
1013 // FIXME: A header-name could be formed starting or ending with an
1014 // alternative token. It's not clear whether that's ill-formed in all
1015 // cases.
1016 while (FilenameTok.isNot(tok::greater)) {
1017 Lex(FilenameTok);
1018 if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
1019 Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
1020 Diag(Start, diag::note_matching) << tok::less;
1021 return true;
1022 }
1023
1024 End = FilenameTok.getLocation();
1025
1026 // FIXME: Provide code completion for #includes.
1027 if (FilenameTok.is(tok::code_completion)) {
1029 Lex(FilenameTok);
1030 continue;
1031 }
1032
1033 // Append the spelling of this token to the buffer. If there was a space
1034 // before it, add it now.
1035 if (FilenameTok.hasLeadingSpace())
1036 FilenameBuffer.push_back(' ');
1037
1038 // Get the spelling of the token, directly into FilenameBuffer if
1039 // possible.
1040 size_t PreAppendSize = FilenameBuffer.size();
1041 FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
1042
1043 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1044 unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
1045
1046 // If the token was spelled somewhere else, copy it into FilenameBuffer.
1047 if (BufPtr != &FilenameBuffer[PreAppendSize])
1048 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1049
1050 // Resize FilenameBuffer to the correct size.
1051 if (FilenameTok.getLength() != ActualLen)
1052 FilenameBuffer.resize(PreAppendSize + ActualLen);
1053 }
1054
1055 FilenameTok.startToken();
1056 FilenameTok.setKind(tok::header_name);
1057 FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine);
1058 FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace);
1059 FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro);
1060 CreateString(FilenameBuffer, FilenameTok, Start, End);
1061 } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
1062 // Convert a string-literal token of the form " h-char-sequence "
1063 // (produced by macro expansion) into a header-name token.
1064 //
1065 // The rules for header-names don't quite match the rules for
1066 // string-literals, but all the places where they differ result in
1067 // undefined behavior, so we can and do treat them the same.
1068 //
1069 // A string-literal with a prefix or suffix is not translated into a
1070 // header-name. This could theoretically be observable via the C++20
1071 // context-sensitive header-name formation rules.
1072 StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
1073 if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1074 FilenameTok.setKind(tok::header_name);
1075 }
1076
1077 return false;
1078}
1079
1080/// Collect the tokens of a C++20 pp-import-suffix.
1082 // FIXME: For error recovery, consider recognizing attribute syntax here
1083 // and terminating / diagnosing a missing semicolon if we find anything
1084 // else? (Can we leave that to the parser?)
1085 unsigned BracketDepth = 0;
1086 while (true) {
1087 Toks.emplace_back();
1088 Lex(Toks.back());
1089
1090 switch (Toks.back().getKind()) {
1091 case tok::l_paren: case tok::l_square: case tok::l_brace:
1092 ++BracketDepth;
1093 break;
1094
1095 case tok::r_paren: case tok::r_square: case tok::r_brace:
1096 if (BracketDepth == 0)
1097 return;
1098 --BracketDepth;
1099 break;
1100
1101 case tok::semi:
1102 if (BracketDepth == 0)
1103 return;
1104 break;
1105
1106 case tok::eof:
1107 return;
1108
1109 default:
1110 break;
1111 }
1112 }
1113}
1114
1115
1116/// Lex a token following the 'import' contextual keyword.
1117///
1118/// pp-import: [C++20]
1119/// import header-name pp-import-suffix[opt] ;
1120/// import header-name-tokens pp-import-suffix[opt] ;
1121/// [ObjC] @ import module-name ;
1122/// [Clang] import module-name ;
1123///
1124/// header-name-tokens:
1125/// string-literal
1126/// < [any sequence of preprocessing-tokens other than >] >
1127///
1128/// module-name:
1129/// module-name-qualifier[opt] identifier
1130///
1131/// module-name-qualifier
1132/// module-name-qualifier[opt] identifier .
1133///
1134/// We respond to a pp-import by importing macros from the named module.
1136 // Figure out what kind of lexer we actually have.
1138
1139 // Lex the next token. The header-name lexing rules are used at the start of
1140 // a pp-import.
1141 //
1142 // For now, we only support header-name imports in C++20 mode.
1143 // FIXME: Should we allow this in all language modes that support an import
1144 // declaration as an extension?
1145 if (NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1146 if (LexHeaderName(Result))
1147 return true;
1148
1149 if (Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
1150 std::string Name = ModuleDeclState.getPrimaryName().str();
1151 Name += ":";
1152 NamedModuleImportPath.push_back(
1153 {getIdentifierInfo(Name), Result.getLocation()});
1154 CurLexerCallback = CLK_LexAfterModuleImport;
1155 return true;
1156 }
1157 } else {
1158 Lex(Result);
1159 }
1160
1161 // Allocate a holding buffer for a sequence of tokens and introduce it into
1162 // the token stream.
1163 auto EnterTokens = [this](ArrayRef<Token> Toks) {
1164 auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1165 std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1166 EnterTokenStream(std::move(ToksCopy), Toks.size(),
1167 /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1168 };
1169
1170 bool ImportingHeader = Result.is(tok::header_name);
1171 // Check for a header-name.
1173 if (ImportingHeader) {
1174 // Enter the header-name token into the token stream; a Lex action cannot
1175 // both return a token and cache tokens (doing so would corrupt the token
1176 // cache if the call to Lex comes from CachingLex / PeekAhead).
1177 Suffix.push_back(Result);
1178
1179 // Consume the pp-import-suffix and expand any macros in it now. We'll add
1180 // it back into the token stream later.
1181 CollectPpImportSuffix(Suffix);
1182 if (Suffix.back().isNot(tok::semi)) {
1183 // This is not a pp-import after all.
1184 EnterTokens(Suffix);
1185 return false;
1186 }
1187
1188 // C++2a [cpp.module]p1:
1189 // The ';' preprocessing-token terminating a pp-import shall not have
1190 // been produced by macro replacement.
1191 SourceLocation SemiLoc = Suffix.back().getLocation();
1192 if (SemiLoc.isMacroID())
1193 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1194
1195 // Reconstitute the import token.
1196 Token ImportTok;
1197 ImportTok.startToken();
1198 ImportTok.setKind(tok::kw_import);
1199 ImportTok.setLocation(ModuleImportLoc);
1200 ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
1201 ImportTok.setLength(6);
1202
1203 auto Action = HandleHeaderIncludeOrImport(
1204 /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
1205 switch (Action.Kind) {
1206 case ImportAction::None:
1207 break;
1208
1209 case ImportAction::ModuleBegin:
1210 // Let the parser know we're textually entering the module.
1211 Suffix.emplace_back();
1212 Suffix.back().startToken();
1213 Suffix.back().setKind(tok::annot_module_begin);
1214 Suffix.back().setLocation(SemiLoc);
1215 Suffix.back().setAnnotationEndLoc(SemiLoc);
1216 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1217 [[fallthrough]];
1218
1219 case ImportAction::ModuleImport:
1220 case ImportAction::HeaderUnitImport:
1221 case ImportAction::SkippedModuleImport:
1222 // We chose to import (or textually enter) the file. Convert the
1223 // header-name token into a header unit annotation token.
1224 Suffix[0].setKind(tok::annot_header_unit);
1225 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1226 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1227 // FIXME: Call the moduleImport callback?
1228 break;
1229 case ImportAction::Failure:
1230 assert(TheModuleLoader.HadFatalFailure &&
1231 "This should be an early exit only to a fatal error");
1232 Result.setKind(tok::eof);
1233 CurLexer->cutOffLexing();
1234 EnterTokens(Suffix);
1235 return true;
1236 }
1237
1238 EnterTokens(Suffix);
1239 return false;
1240 }
1241
1242 // The token sequence
1243 //
1244 // import identifier (. identifier)*
1245 //
1246 // indicates a module import directive. We already saw the 'import'
1247 // contextual keyword, so now we're looking for the identifiers.
1248 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1249 // We expected to see an identifier here, and we did; continue handling
1250 // identifiers.
1251 NamedModuleImportPath.push_back(
1252 std::make_pair(Result.getIdentifierInfo(), Result.getLocation()));
1253 ModuleImportExpectsIdentifier = false;
1254 CurLexerCallback = CLK_LexAfterModuleImport;
1255 return true;
1256 }
1257
1258 // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1259 // see the next identifier. (We can also see a '[[' that begins an
1260 // attribute-specifier-seq here under the Standard C++ Modules.)
1261 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1262 ModuleImportExpectsIdentifier = true;
1263 CurLexerCallback = CLK_LexAfterModuleImport;
1264 return true;
1265 }
1266
1267 // If we didn't recognize a module name at all, this is not a (valid) import.
1268 if (NamedModuleImportPath.empty() || Result.is(tok::eof))
1269 return true;
1270
1271 // Consume the pp-import-suffix and expand any macros in it now, if we're not
1272 // at the semicolon already.
1273 SourceLocation SemiLoc = Result.getLocation();
1274 if (Result.isNot(tok::semi)) {
1275 Suffix.push_back(Result);
1276 CollectPpImportSuffix(Suffix);
1277 if (Suffix.back().isNot(tok::semi)) {
1278 // This is not an import after all.
1279 EnterTokens(Suffix);
1280 return false;
1281 }
1282 SemiLoc = Suffix.back().getLocation();
1283 }
1284
1285 // Under the standard C++ Modules, the dot is just part of the module name,
1286 // and not a real hierarchy separator. Flatten such module names now.
1287 //
1288 // FIXME: Is this the right level to be performing this transformation?
1289 std::string FlatModuleName;
1290 if (getLangOpts().CPlusPlusModules) {
1291 for (auto &Piece : NamedModuleImportPath) {
1292 // If the FlatModuleName ends with colon, it implies it is a partition.
1293 if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
1294 FlatModuleName += ".";
1295 FlatModuleName += Piece.first->getName();
1296 }
1297 SourceLocation FirstPathLoc = NamedModuleImportPath[0].second;
1298 NamedModuleImportPath.clear();
1299 NamedModuleImportPath.push_back(
1300 std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1301 }
1302
1303 Module *Imported = nullptr;
1304 // We don't/shouldn't load the standard c++20 modules when preprocessing.
1305 if (getLangOpts().Modules && !isInImportingCXXNamedModules()) {
1306 Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1307 NamedModuleImportPath,
1309 /*IsInclusionDirective=*/false);
1310 if (Imported)
1311 makeModuleVisible(Imported, SemiLoc);
1312 }
1313
1314 if (Callbacks)
1315 Callbacks->moduleImport(ModuleImportLoc, NamedModuleImportPath, Imported);
1316
1317 if (!Suffix.empty()) {
1318 EnterTokens(Suffix);
1319 return false;
1320 }
1321 return true;
1322}
1323
1325 CurSubmoduleState->VisibleModules.setVisible(
1326 M, Loc, [](Module *) {},
1327 [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1328 // FIXME: Include the path in the diagnostic.
1329 // FIXME: Include the import location for the conflicting module.
1330 Diag(ModuleImportLoc, diag::warn_module_conflict)
1331 << Path[0]->getFullModuleName()
1332 << Conflict->getFullModuleName()
1333 << Message;
1334 });
1335
1336 // Add this module to the imports list of the currently-built submodule.
1337 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1338 BuildingSubmoduleStack.back().M->Imports.insert(M);
1339}
1340
1342 const char *DiagnosticTag,
1343 bool AllowMacroExpansion) {
1344 // We need at least one string literal.
1345 if (Result.isNot(tok::string_literal)) {
1346 Diag(Result, diag::err_expected_string_literal)
1347 << /*Source='in...'*/0 << DiagnosticTag;
1348 return false;
1349 }
1350
1351 // Lex string literal tokens, optionally with macro expansion.
1352 SmallVector<Token, 4> StrToks;
1353 do {
1354 StrToks.push_back(Result);
1355
1356 if (Result.hasUDSuffix())
1357 Diag(Result, diag::err_invalid_string_udl);
1358
1359 if (AllowMacroExpansion)
1360 Lex(Result);
1361 else
1363 } while (Result.is(tok::string_literal));
1364
1365 // Concatenate and parse the strings.
1366 StringLiteralParser Literal(StrToks, *this);
1367 assert(Literal.isOrdinary() && "Didn't allow wide strings in");
1368
1369 if (Literal.hadError)
1370 return false;
1371
1372 if (Literal.Pascal) {
1373 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1374 << /*Source='in...'*/0 << DiagnosticTag;
1375 return false;
1376 }
1377
1378 String = std::string(Literal.GetString());
1379 return true;
1380}
1381
1383 assert(Tok.is(tok::numeric_constant));
1384 SmallString<8> IntegerBuffer;
1385 bool NumberInvalid = false;
1386 StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1387 if (NumberInvalid)
1388 return false;
1389 NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
1391 getDiagnostics());
1392 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1393 return false;
1394 llvm::APInt APVal(64, 0);
1395 if (Literal.GetIntegerValue(APVal))
1396 return false;
1397 Lex(Tok);
1398 Value = APVal.getLimitedValue();
1399 return true;
1400}
1401
1403 assert(Handler && "NULL comment handler");
1404 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1405 "Comment handler already registered");
1406 CommentHandlers.push_back(Handler);
1407}
1408
1410 std::vector<CommentHandler *>::iterator Pos =
1411 llvm::find(CommentHandlers, Handler);
1412 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1413 CommentHandlers.erase(Pos);
1414}
1415
1417 bool AnyPendingTokens = false;
1418 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1419 HEnd = CommentHandlers.end();
1420 H != HEnd; ++H) {
1421 if ((*H)->HandleComment(*this, Comment))
1422 AnyPendingTokens = true;
1423 }
1424 if (!AnyPendingTokens || getCommentRetentionState())
1425 return false;
1426 Lex(result);
1427 return true;
1428}
1429
1430void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1431 const MacroAnnotations &A =
1432 getMacroAnnotations(Identifier.getIdentifierInfo());
1433 assert(A.DeprecationInfo &&
1434 "Macro deprecation warning without recorded annotation!");
1435 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1436 if (Info.Message.empty())
1437 Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1438 << Identifier.getIdentifierInfo() << 0;
1439 else
1440 Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1441 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1442 Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1443}
1444
1445void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1446 const MacroAnnotations &A =
1447 getMacroAnnotations(Identifier.getIdentifierInfo());
1448 assert(A.RestrictExpansionInfo &&
1449 "Macro restricted expansion warning without recorded annotation!");
1450 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1451 if (Info.Message.empty())
1452 Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1453 << Identifier.getIdentifierInfo() << 0;
1454 else
1455 Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1456 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1457 Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1458}
1459
1460void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1461 bool IsUndef) const {
1462 const MacroAnnotations &A =
1463 getMacroAnnotations(Identifier.getIdentifierInfo());
1464 assert(A.FinalAnnotationLoc &&
1465 "Final macro warning without recorded annotation!");
1466
1467 Diag(Identifier, diag::warn_pragma_final_macro)
1468 << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1469 Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1470}
1471
1473 const SourceLocation &Loc) const {
1474 // Try to find a region in `SafeBufferOptOutMap` where `Loc` is in:
1475 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1476 SafeBufferOptOutMap,
1477 [&SourceMgr,
1478 &Loc](const std::pair<SourceLocation, SourceLocation> &Region) {
1479 return SourceMgr.isBeforeInTranslationUnit(Region.second, Loc);
1480 });
1481
1482 if (FirstRegionEndingAfterLoc != SafeBufferOptOutMap.end()) {
1483 // To test if the start location of the found region precedes `Loc`:
1484 return SourceMgr.isBeforeInTranslationUnit(FirstRegionEndingAfterLoc->first,
1485 Loc);
1486 }
1487 // If we do not find a region whose end location passes `Loc`, we want to
1488 // check if the current region is still open:
1489 if (!SafeBufferOptOutMap.empty() &&
1490 SafeBufferOptOutMap.back().first == SafeBufferOptOutMap.back().second)
1491 return SourceMgr.isBeforeInTranslationUnit(SafeBufferOptOutMap.back().first,
1492 Loc);
1493 return false;
1494}
1495
1497 bool isEnter, const SourceLocation &Loc) {
1498 if (isEnter) {
1500 return true; // invalid enter action
1501 InSafeBufferOptOutRegion = true;
1502 CurrentSafeBufferOptOutStart = Loc;
1503
1504 // To set the start location of a new region:
1505
1506 if (!SafeBufferOptOutMap.empty()) {
1507 [[maybe_unused]] auto *PrevRegion = &SafeBufferOptOutMap.back();
1508 assert(PrevRegion->first != PrevRegion->second &&
1509 "Shall not begin a safe buffer opt-out region before closing the "
1510 "previous one.");
1511 }
1512 // If the start location equals to the end location, we call the region a
1513 // open region or a unclosed region (i.e., end location has not been set
1514 // yet).
1515 SafeBufferOptOutMap.emplace_back(Loc, Loc);
1516 } else {
1518 return true; // invalid enter action
1519 InSafeBufferOptOutRegion = false;
1520
1521 // To set the end location of the current open region:
1522
1523 assert(!SafeBufferOptOutMap.empty() &&
1524 "Misordered safe buffer opt-out regions");
1525 auto *CurrRegion = &SafeBufferOptOutMap.back();
1526 assert(CurrRegion->first == CurrRegion->second &&
1527 "Set end location to a closed safe buffer opt-out region");
1528 CurrRegion->second = Loc;
1529 }
1530 return false;
1531}
1532
1534 return InSafeBufferOptOutRegion;
1535}
1537 StartLoc = CurrentSafeBufferOptOutStart;
1538 return InSafeBufferOptOutRegion;
1539}
1540
1541ModuleLoader::~ModuleLoader() = default;
1542
1544
1546
1548
1550 if (Record)
1551 return;
1552
1553 Record = new PreprocessingRecord(getSourceManager());
1554 addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1555}
#define SM(sm)
Definition: Cuda.cpp:80
Defines enum values for all the target-independent builtin functions.
Defines the clang::FileManager interface and associated types.
Defines the FileSystemStatCache interface.
StringRef Identifier
Definition: Format.cpp:2981
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PreprocessorLexer interface.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
Defines the clang::Preprocessor interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive.
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Abstract base class that describes a handler that will receive source ranges for each of the comments...
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...
Definition: FileEntry.h:57
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:366
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isValid() const
bool isInvalid() const
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:232
Module * lookupModule(StringRef ModuleName, SourceLocation ImportLoc=SourceLocation(), bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Provides lookups to, and iteration over, IdentiferInfo objects.
One of these records is kept for each identifier that is lexed.
bool isModulesImport() const
Determine whether this is the contextual keyword import.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setIsFutureCompatKeyword(bool Val)
StringRef getName() const
Return the actual identifier string.
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
bool isExtensionToken() const
get/setExtension - Initialize information about whether or not this language token is an extension.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:83
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Definition: LangOptions.h:307
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
A description of the current definition of a macro.
Definition: MacroInfo.h:590
SourceLocation getLocation() const
Definition: MacroInfo.h:488
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
const_tokens_iterator tokens_begin() const
Definition: MacroInfo.h:244
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:235
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:237
bool isObjectLike() const
Definition: MacroInfo.h:202
Abstract interface for a module loader.
Definition: ModuleLoader.h:82
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
virtual ~ModuleLoader()
Represents a macro directive exported by a module.
Definition: MacroInfo.h:514
Describes a module or submodule.
Definition: Module.h:105
@ Hidden
All of the names in this module are hidden.
Definition: Module.h:388
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber,...
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:96
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
bool markIncluded(FileEntryRef File)
Mark the file as included.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
void DumpMacro(const MacroInfo &MI) const
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool SetCodeCompletionPoint(FileEntryRef File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void makeModuleVisible(Module *M, SourceLocation Loc)
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:285
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
bool getCommentRetentionState() const
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
HeaderSearch & getHeaderSearchInfo() const
ExternalPreprocessorSource * getExternalSource() const
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, const LangOptions &LangOpts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void PoisonSEHIdentifiers(bool Poison=true)
size_t getTotalMemory() const
void LexTokensUntilEOF(std::vector< Token > *Tokens=nullptr)
Lex all tokens for this preprocessor until (and excluding) end of file.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
DiagnosticsEngine & getDiagnostics() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Definition: ScratchBuffer.h:24
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
void print(raw_ostream &OS, const SourceManager &SM) const
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
This class handles loading and caching of source files into memory.
OptionalFileEntryRef getFileEntryRefForID(FileID FID) const
Returns the FileEntryRef for the provided FileID.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
FileID getMainFileID() const
Returns the FileID of the main source file.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer.
void overrideFileContents(FileEntryRef SourceFile, const llvm::MemoryBufferRef &Buffer)
Override the contents of the given source file by providing an already-allocated buffer.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned Length, bool ExpansionIsTokenRange=true, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Creates an expansion SLocEntry for a macro use.
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
std::optional< llvm::MemoryBufferRef > getMemoryBufferForFileOrNone(FileEntryRef File)
Retrieve the memory buffer associated with the given file.
A trivial tuple used to represent a source range.
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
Exposes information about the current target.
Definition: TargetInfo.h:212
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:186
void setLiteralData(const char *Ptr)
Definition: Token.h:228
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:303
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:131
unsigned getLength() const
Definition: Token.h:134
void setLength(unsigned Len)
Definition: Token.h:140
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6....
Definition: Token.h:283
void setKind(tok::TokenKind K)
Definition: Token.h:94
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:98
tok::TokenKind getKind() const
Definition: Token.h:93
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:275
@ DisableExpand
Definition: Token.h:78
@ IsReinjected
Definition: Token.h:88
@ LeadingEmptyMacro
Definition: Token.h:80
@ LeadingSpace
Definition: Token.h:76
@ StartOfLine
Definition: Token.h:74
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:279
void setLocation(SourceLocation L)
Definition: Token.h:139
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
Definition: Token.h:296
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:216
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:100
bool isNot(tok::TokenKind K) const
Definition: Token.h:99
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
void startToken()
Reset all flags to cleared.
Definition: Token.h:176
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:292
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:195
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:266
Defines the clang::TargetInfo interface.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:24
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Result
The result type of a method or function.
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:941
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:947
Definition: Format.h:5200
#define true
Definition: stdbool.h:21