clang 17.0.0git
Preprocessor.cpp
Go to the documentation of this file.
1//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Preprocessor interface.
10//
11//===----------------------------------------------------------------------===//
12//
13// Options to support:
14// -H - Print the name of each header file used.
15// -d[DNI] - Dump various things.
16// -fworking-directory - #line's with preprocessor's working dir.
17// -fpreprocessed
18// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19// -W*
20// -w
21//
22// Messages to emit:
23// "Multiple include guards may be useful for:\n"
24//
25//===----------------------------------------------------------------------===//
26
32#include "clang/Basic/LLVM.h"
34#include "clang/Basic/Module.h"
42#include "clang/Lex/Lexer.h"
44#include "clang/Lex/MacroArgs.h"
45#include "clang/Lex/MacroInfo.h"
47#include "clang/Lex/Pragma.h"
52#include "clang/Lex/Token.h"
54#include "llvm/ADT/APInt.h"
55#include "llvm/ADT/ArrayRef.h"
56#include "llvm/ADT/DenseMap.h"
57#include "llvm/ADT/STLExtras.h"
58#include "llvm/ADT/SmallString.h"
59#include "llvm/ADT/SmallVector.h"
60#include "llvm/ADT/StringRef.h"
61#include "llvm/Support/Capacity.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/MemoryBuffer.h"
64#include "llvm/Support/raw_ostream.h"
65#include <algorithm>
66#include <cassert>
67#include <memory>
68#include <optional>
69#include <string>
70#include <utility>
71#include <vector>
72
73using namespace clang;
74
75LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
76
78
79Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
80 DiagnosticsEngine &diags, LangOptions &opts,
81 SourceManager &SM, HeaderSearch &Headers,
82 ModuleLoader &TheModuleLoader,
83 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
85 : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
87 ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
89 // As the language options may have not been loaded yet (when
90 // deserializing an ASTUnit), adding keywords to the identifier table is
91 // deferred to Preprocessor::Initialize().
92 Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
93 TUKind(TUKind), SkipMainFilePreamble(0, true),
94 CurSubmoduleState(&NullSubmoduleState) {
95 OwnsHeaderSearch = OwnsHeaders;
96
97 // Default to discarding comments.
98 KeepComments = false;
99 KeepMacroComments = false;
100 SuppressIncludeNotFoundError = false;
101
102 // Macro expansion is enabled.
103 DisableMacroExpansion = false;
104 MacroExpansionInDirectivesOverride = false;
105 InMacroArgs = false;
106 ArgMacro = nullptr;
107 InMacroArgPreExpansion = false;
108 NumCachedTokenLexers = 0;
109 PragmasEnabled = true;
110 ParsingIfOrElifDirective = false;
111 PreprocessedOutput = false;
112
113 // We haven't read anything from the external source.
114 ReadMacrosFromExternalSource = false;
115
116 BuiltinInfo = std::make_unique<Builtin::Context>();
117
118 // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
119 // a macro. They get unpoisoned where it is allowed.
120 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
121 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
122 (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
123 SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
124
125 // Initialize the pragma handlers.
126 RegisterBuiltinPragmas();
127
128 // Initialize builtin macros like __LINE__ and friends.
129 RegisterBuiltinMacros();
130
131 if(LangOpts.Borland) {
132 Ident__exception_info = getIdentifierInfo("_exception_info");
133 Ident___exception_info = getIdentifierInfo("__exception_info");
134 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
135 Ident__exception_code = getIdentifierInfo("_exception_code");
136 Ident___exception_code = getIdentifierInfo("__exception_code");
137 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
138 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
139 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
140 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
141 } else {
142 Ident__exception_info = Ident__exception_code = nullptr;
143 Ident__abnormal_termination = Ident___exception_info = nullptr;
144 Ident___exception_code = Ident___abnormal_termination = nullptr;
145 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
146 Ident_AbnormalTermination = nullptr;
147 }
148
149 // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
151 SkippingUntilPragmaHdrStop = true;
152
153 // If using a PCH with a through header, start skipping tokens.
154 if (!this->PPOpts->PCHThroughHeader.empty() &&
155 !this->PPOpts->ImplicitPCHInclude.empty())
156 SkippingUntilPCHThroughHeader = true;
157
158 if (this->PPOpts->GeneratePreamble)
159 PreambleConditionalStack.startRecording();
160
161 MaxTokens = LangOpts.MaxTokens;
162}
163
165 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
166
167 IncludeMacroStack.clear();
168
169 // Free any cached macro expanders.
170 // This populates MacroArgCache, so all TokenLexers need to be destroyed
171 // before the code below that frees up the MacroArgCache list.
172 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
173 CurTokenLexer.reset();
174
175 // Free any cached MacroArgs.
176 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
177 ArgList = ArgList->deallocate();
178
179 // Delete the header search info, if we own it.
180 if (OwnsHeaderSearch)
181 delete &HeaderInfo;
182}
183
185 const TargetInfo *AuxTarget) {
186 assert((!this->Target || this->Target == &Target) &&
187 "Invalid override of target information");
188 this->Target = &Target;
189
190 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
191 "Invalid override of aux target information.");
192 this->AuxTarget = AuxTarget;
193
194 // Initialize information about built-ins.
195 BuiltinInfo->InitializeTarget(Target, AuxTarget);
196 HeaderInfo.setTarget(Target);
197
198 // Populate the identifier table with info about keywords for the current language.
199 Identifiers.AddKeywords(LangOpts);
200
201 // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
202 setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
203
204 if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine)
205 // Use setting from TargetInfo.
206 setCurrentFPEvalMethod(SourceLocation(), Target.getFPEvalMethod());
207 else
208 // Set initial value of __FLT_EVAL_METHOD__ from the command line.
209 setCurrentFPEvalMethod(SourceLocation(), getLangOpts().getFPEvalMethod());
210}
211
213 NumEnteredSourceFiles = 0;
214
215 // Reset pragmas
216 PragmaHandlersBackup = std::move(PragmaHandlers);
217 PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
218 RegisterBuiltinPragmas();
219
220 // Reset PredefinesFileID
221 PredefinesFileID = FileID();
222}
223
225 NumEnteredSourceFiles = 1;
226
227 PragmaHandlers = std::move(PragmaHandlersBackup);
228}
229
230void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
231 llvm::errs() << tok::getTokenName(Tok.getKind());
232
233 if (!Tok.isAnnotation())
234 llvm::errs() << " '" << getSpelling(Tok) << "'";
235
236 if (!DumpFlags) return;
237
238 llvm::errs() << "\t";
239 if (Tok.isAtStartOfLine())
240 llvm::errs() << " [StartOfLine]";
241 if (Tok.hasLeadingSpace())
242 llvm::errs() << " [LeadingSpace]";
243 if (Tok.isExpandDisabled())
244 llvm::errs() << " [ExpandDisabled]";
245 if (Tok.needsCleaning()) {
246 const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
247 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
248 << "']";
249 }
250
251 llvm::errs() << "\tLoc=<";
253 llvm::errs() << ">";
254}
255
257 Loc.print(llvm::errs(), SourceMgr);
258}
259
260void Preprocessor::DumpMacro(const MacroInfo &MI) const {
261 llvm::errs() << "MACRO: ";
262 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
264 llvm::errs() << " ";
265 }
266 llvm::errs() << "\n";
267}
268
270 llvm::errs() << "\n*** Preprocessor Stats:\n";
271 llvm::errs() << NumDirectives << " directives found:\n";
272 llvm::errs() << " " << NumDefined << " #define.\n";
273 llvm::errs() << " " << NumUndefined << " #undef.\n";
274 llvm::errs() << " #include/#include_next/#import:\n";
275 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
276 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
277 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
278 llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
279 llvm::errs() << " " << NumEndif << " #endif.\n";
280 llvm::errs() << " " << NumPragma << " #pragma.\n";
281 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
282
283 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
284 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
285 << NumFastMacroExpanded << " on the fast path.\n";
286 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
287 << " token paste (##) operations performed, "
288 << NumFastTokenPaste << " on the fast path.\n";
289
290 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
291
292 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
293 llvm::errs() << "\n Macro Expanded Tokens: "
294 << llvm::capacity_in_bytes(MacroExpandedTokens);
295 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
296 // FIXME: List information for all submodules.
297 llvm::errs() << "\n Macros: "
298 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
299 llvm::errs() << "\n #pragma push_macro Info: "
300 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
301 llvm::errs() << "\n Poison Reasons: "
302 << llvm::capacity_in_bytes(PoisonReasons);
303 llvm::errs() << "\n Comment Handlers: "
304 << llvm::capacity_in_bytes(CommentHandlers) << "\n";
305}
306
308Preprocessor::macro_begin(bool IncludeExternalMacros) const {
309 if (IncludeExternalMacros && ExternalSource &&
310 !ReadMacrosFromExternalSource) {
311 ReadMacrosFromExternalSource = true;
312 ExternalSource->ReadDefinedMacros();
313 }
314
315 // Make sure we cover all macros in visible modules.
316 for (const ModuleMacro &Macro : ModuleMacros)
317 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
318
319 return CurSubmoduleState->Macros.begin();
320}
321
323 return BP.getTotalMemory()
324 + llvm::capacity_in_bytes(MacroExpandedTokens)
325 + Predefines.capacity() /* Predefines buffer. */
326 // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
327 // and ModuleMacros.
328 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
329 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
330 + llvm::capacity_in_bytes(PoisonReasons)
331 + llvm::capacity_in_bytes(CommentHandlers);
332}
333
335Preprocessor::macro_end(bool IncludeExternalMacros) const {
336 if (IncludeExternalMacros && ExternalSource &&
337 !ReadMacrosFromExternalSource) {
338 ReadMacrosFromExternalSource = true;
339 ExternalSource->ReadDefinedMacros();
340 }
341
342 return CurSubmoduleState->Macros.end();
343}
344
345/// Compares macro tokens with a specified token value sequence.
346static bool MacroDefinitionEquals(const MacroInfo *MI,
347 ArrayRef<TokenValue> Tokens) {
348 return Tokens.size() == MI->getNumTokens() &&
349 std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
350}
351
353 SourceLocation Loc,
354 ArrayRef<TokenValue> Tokens) const {
355 SourceLocation BestLocation;
356 StringRef BestSpelling;
358 I != E; ++I) {
360 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
361 if (!Def || !Def.getMacroInfo())
362 continue;
363 if (!Def.getMacroInfo()->isObjectLike())
364 continue;
365 if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
366 continue;
367 SourceLocation Location = Def.getLocation();
368 // Choose the macro defined latest.
369 if (BestLocation.isInvalid() ||
370 (Location.isValid() &&
371 SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
372 BestLocation = Location;
373 BestSpelling = I->first->getName();
374 }
375 }
376 return BestSpelling;
377}
378
380 if (CurLexer)
381 CurLexerKind = CurLexer->isDependencyDirectivesLexer()
382 ? CLK_DependencyDirectivesLexer
383 : CLK_Lexer;
384 else if (CurTokenLexer)
385 CurLexerKind = CLK_TokenLexer;
386 else
387 CurLexerKind = CLK_CachingLexer;
388}
389
391 unsigned CompleteLine,
392 unsigned CompleteColumn) {
393 assert(File);
394 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
395 assert(!CodeCompletionFile && "Already set");
396
397 // Load the actual file's contents.
398 std::optional<llvm::MemoryBufferRef> Buffer =
400 if (!Buffer)
401 return true;
402
403 // Find the byte position of the truncation point.
404 const char *Position = Buffer->getBufferStart();
405 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
406 for (; *Position; ++Position) {
407 if (*Position != '\r' && *Position != '\n')
408 continue;
409
410 // Eat \r\n or \n\r as a single line.
411 if ((Position[1] == '\r' || Position[1] == '\n') &&
412 Position[0] != Position[1])
413 ++Position;
414 ++Position;
415 break;
416 }
417 }
418
419 Position += CompleteColumn - 1;
420
421 // If pointing inside the preamble, adjust the position at the beginning of
422 // the file after the preamble.
423 if (SkipMainFilePreamble.first &&
424 SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
425 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
426 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
427 }
428
429 if (Position > Buffer->getBufferEnd())
430 Position = Buffer->getBufferEnd();
431
432 CodeCompletionFile = File;
433 CodeCompletionOffset = Position - Buffer->getBufferStart();
434
435 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
436 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
437 char *NewBuf = NewBuffer->getBufferStart();
438 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
439 *NewPos = '\0';
440 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
441 SourceMgr.overrideFileContents(File, std::move(NewBuffer));
442
443 return false;
444}
445
447 bool IsAngled) {
449 if (CodeComplete)
450 CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
451}
452
455 if (CodeComplete)
456 CodeComplete->CodeCompleteNaturalLanguage();
457}
458
459/// getSpelling - This method is used to get the spelling of a token into a
460/// SmallVector. Note that the returned StringRef may not point to the
461/// supplied buffer if a copy can be avoided.
462StringRef Preprocessor::getSpelling(const Token &Tok,
463 SmallVectorImpl<char> &Buffer,
464 bool *Invalid) const {
465 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
466 if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
467 // Try the fast path.
468 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
469 return II->getName();
470 }
471
472 // Resize the buffer if we need to copy into it.
473 if (Tok.needsCleaning())
474 Buffer.resize(Tok.getLength());
475
476 const char *Ptr = Buffer.data();
477 unsigned Len = getSpelling(Tok, Ptr, Invalid);
478 return StringRef(Ptr, Len);
479}
480
481/// CreateString - Plop the specified string into a scratch buffer and return a
482/// location for it. If specified, the source location provides a source
483/// location for the token.
484void Preprocessor::CreateString(StringRef Str, Token &Tok,
485 SourceLocation ExpansionLocStart,
486 SourceLocation ExpansionLocEnd) {
487 Tok.setLength(Str.size());
488
489 const char *DestPtr;
490 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
491
492 if (ExpansionLocStart.isValid())
493 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
494 ExpansionLocEnd, Str.size());
495 Tok.setLocation(Loc);
496
497 // If this is a raw identifier or a literal token, set the pointer data.
498 if (Tok.is(tok::raw_identifier))
499 Tok.setRawIdentifierData(DestPtr);
500 else if (Tok.isLiteral())
501 Tok.setLiteralData(DestPtr);
502}
503
505 auto &SM = getSourceManager();
506 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
507 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
508 bool Invalid = false;
509 StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
510 if (Invalid)
511 return SourceLocation();
512
513 // FIXME: We could consider re-using spelling for tokens we see repeatedly.
514 const char *DestPtr;
515 SourceLocation Spelling =
516 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
517 return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
518}
519
521 if (!getLangOpts().isCompilingModule())
522 return nullptr;
523
524 return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
525}
526
528 if (!getLangOpts().isCompilingModuleImplementation())
529 return nullptr;
530
531 return getHeaderSearchInfo().lookupModule(getLangOpts().ModuleName);
532}
533
534//===----------------------------------------------------------------------===//
535// Preprocessor Initialization Methods
536//===----------------------------------------------------------------------===//
537
538/// EnterMainSourceFile - Enter the specified FileID as the main source file,
539/// which implicitly adds the builtin defines etc.
541 // We do not allow the preprocessor to reenter the main file. Doing so will
542 // cause FileID's to accumulate information from both runs (e.g. #line
543 // information) and predefined macros aren't guaranteed to be set properly.
544 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
545 FileID MainFileID = SourceMgr.getMainFileID();
546
547 // If MainFileID is loaded it means we loaded an AST file, no need to enter
548 // a main file.
549 if (!SourceMgr.isLoadedFileID(MainFileID)) {
550 // Enter the main file source buffer.
551 EnterSourceFile(MainFileID, nullptr, SourceLocation());
552
553 // If we've been asked to skip bytes in the main file (e.g., as part of a
554 // precompiled preamble), do so now.
555 if (SkipMainFilePreamble.first > 0)
556 CurLexer->SetByteOffset(SkipMainFilePreamble.first,
557 SkipMainFilePreamble.second);
558
559 // Tell the header info that the main file was entered. If the file is later
560 // #imported, it won't be re-entered.
561 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
562 markIncluded(FE);
563 }
564
565 // Preprocess Predefines to populate the initial preprocessor state.
566 std::unique_ptr<llvm::MemoryBuffer> SB =
567 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
568 assert(SB && "Cannot create predefined source buffer");
569 FileID FID = SourceMgr.createFileID(std::move(SB));
570 assert(FID.isValid() && "Could not create FileID for predefines?");
571 setPredefinesFileID(FID);
572
573 // Start parsing the predefines.
574 EnterSourceFile(FID, nullptr, SourceLocation());
575
576 if (!PPOpts->PCHThroughHeader.empty()) {
577 // Lookup and save the FileID for the through header. If it isn't found
578 // in the search path, it's a fatal error.
580 SourceLocation(), PPOpts->PCHThroughHeader,
581 /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
582 /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
583 /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
584 /*IsFrameworkFound=*/nullptr);
585 if (!File) {
586 Diag(SourceLocation(), diag::err_pp_through_header_not_found)
587 << PPOpts->PCHThroughHeader;
588 return;
589 }
590 setPCHThroughHeaderFileID(
592 }
593
594 // Skip tokens from the Predefines and if needed the main file.
595 if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
596 (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
598}
599
600void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
601 assert(PCHThroughHeaderFileID.isInvalid() &&
602 "PCHThroughHeaderFileID already set!");
603 PCHThroughHeaderFileID = FID;
604}
605
607 assert(PCHThroughHeaderFileID.isValid() &&
608 "Invalid PCH through header FileID");
609 return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
610}
611
613 return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
614 PCHThroughHeaderFileID.isValid();
615}
616
618 return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
619 PCHThroughHeaderFileID.isValid();
620}
621
623 return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop;
624}
625
627 return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop;
628}
629
630/// Skip tokens until after the #include of the through header or
631/// until after a #pragma hdrstop is seen. Tokens in the predefines file
632/// and the main file may be skipped. If the end of the predefines file
633/// is reached, skipping continues into the main file. If the end of the
634/// main file is reached, it's a fatal error.
636 bool ReachedMainFileEOF = false;
637 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
638 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
639 Token Tok;
640 while (true) {
641 bool InPredefines =
642 (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
643 switch (CurLexerKind) {
644 case CLK_Lexer:
645 CurLexer->Lex(Tok);
646 break;
647 case CLK_TokenLexer:
648 CurTokenLexer->Lex(Tok);
649 break;
650 case CLK_CachingLexer:
651 CachingLex(Tok);
652 break;
653 case CLK_DependencyDirectivesLexer:
654 CurLexer->LexDependencyDirectiveToken(Tok);
655 break;
656 case CLK_LexAfterModuleImport:
658 break;
659 }
660 if (Tok.is(tok::eof) && !InPredefines) {
661 ReachedMainFileEOF = true;
662 break;
663 }
664 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
665 break;
666 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
667 break;
668 }
669 if (ReachedMainFileEOF) {
670 if (UsingPCHThroughHeader)
671 Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
672 << PPOpts->PCHThroughHeader << 1;
673 else if (!PPOpts->PCHWithHdrStopCreate)
674 Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
675 }
676}
677
678void Preprocessor::replayPreambleConditionalStack() {
679 // Restore the conditional stack from the preamble, if there is one.
680 if (PreambleConditionalStack.isReplaying()) {
681 assert(CurPPLexer &&
682 "CurPPLexer is null when calling replayPreambleConditionalStack.");
683 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
684 PreambleConditionalStack.doneReplaying();
685 if (PreambleConditionalStack.reachedEOFWhileSkipping())
686 SkipExcludedConditionalBlock(
687 PreambleConditionalStack.SkipInfo->HashTokenLoc,
688 PreambleConditionalStack.SkipInfo->IfTokenLoc,
689 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
690 PreambleConditionalStack.SkipInfo->FoundElse,
691 PreambleConditionalStack.SkipInfo->ElseLoc);
692 }
693}
694
696 // Notify the client that we reached the end of the source file.
697 if (Callbacks)
698 Callbacks->EndOfMainFile();
699}
700
701//===----------------------------------------------------------------------===//
702// Lexer Event Handling.
703//===----------------------------------------------------------------------===//
704
705/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
706/// identifier information for the token and install it into the token,
707/// updating the token kind accordingly.
709 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
710
711 // Look up this token, see if it is a macro, or if it is a language keyword.
712 IdentifierInfo *II;
713 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
714 // No cleaning needed, just use the characters from the lexed buffer.
715 II = getIdentifierInfo(Identifier.getRawIdentifier());
716 } else {
717 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
718 SmallString<64> IdentifierBuffer;
719 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
720
721 if (Identifier.hasUCN()) {
722 SmallString<64> UCNIdentifierBuffer;
723 expandUCNs(UCNIdentifierBuffer, CleanedStr);
724 II = getIdentifierInfo(UCNIdentifierBuffer);
725 } else {
726 II = getIdentifierInfo(CleanedStr);
727 }
728 }
729
730 // Update the token info (identifier info and appropriate token kind).
731 // FIXME: the raw_identifier may contain leading whitespace which is removed
732 // from the cleaned identifier token. The SourceLocation should be updated to
733 // refer to the non-whitespace character. For instance, the text "\\\nB" (a
734 // line continuation before 'B') is parsed as a single tok::raw_identifier and
735 // is cleaned to tok::identifier "B". After cleaning the token's length is
736 // still 3 and the SourceLocation refers to the location of the backslash.
737 Identifier.setIdentifierInfo(II);
738 Identifier.setKind(II->getTokenID());
739
740 return II;
741}
742
744 PoisonReasons[II] = DiagID;
745}
746
748 assert(Ident__exception_code && Ident__exception_info);
749 assert(Ident___exception_code && Ident___exception_info);
750 Ident__exception_code->setIsPoisoned(Poison);
751 Ident___exception_code->setIsPoisoned(Poison);
752 Ident_GetExceptionCode->setIsPoisoned(Poison);
753 Ident__exception_info->setIsPoisoned(Poison);
754 Ident___exception_info->setIsPoisoned(Poison);
755 Ident_GetExceptionInfo->setIsPoisoned(Poison);
756 Ident__abnormal_termination->setIsPoisoned(Poison);
757 Ident___abnormal_termination->setIsPoisoned(Poison);
758 Ident_AbnormalTermination->setIsPoisoned(Poison);
759}
760
762 assert(Identifier.getIdentifierInfo() &&
763 "Can't handle identifiers without identifier info!");
764 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
765 PoisonReasons.find(Identifier.getIdentifierInfo());
766 if(it == PoisonReasons.end())
767 Diag(Identifier, diag::err_pp_used_poisoned_id);
768 else
769 Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
770}
771
772void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
773 assert(II.isOutOfDate() && "not out of date");
775}
776
777/// HandleIdentifier - This callback is invoked when the lexer reads an
778/// identifier. This callback looks up the identifier in the map and/or
779/// potentially macro expands it or turns it into a named token (like 'for').
780///
781/// Note that callers of this method are guarded by checking the
782/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
783/// IdentifierInfo methods that compute these properties will need to change to
784/// match.
786 assert(Identifier.getIdentifierInfo() &&
787 "Can't handle identifiers without identifier info!");
788
789 IdentifierInfo &II = *Identifier.getIdentifierInfo();
790
791 // If the information about this identifier is out of date, update it from
792 // the external source.
793 // We have to treat __VA_ARGS__ in a special way, since it gets
794 // serialized with isPoisoned = true, but our preprocessor may have
795 // unpoisoned it if we're defining a C99 macro.
796 if (II.isOutOfDate()) {
797 bool CurrentIsPoisoned = false;
798 const bool IsSpecialVariadicMacro =
799 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
800 if (IsSpecialVariadicMacro)
801 CurrentIsPoisoned = II.isPoisoned();
802
803 updateOutOfDateIdentifier(II);
804 Identifier.setKind(II.getTokenID());
805
806 if (IsSpecialVariadicMacro)
807 II.setIsPoisoned(CurrentIsPoisoned);
808 }
809
810 // If this identifier was poisoned, and if it was not produced from a macro
811 // expansion, emit an error.
812 if (II.isPoisoned() && CurPPLexer) {
814 }
815
816 // If this is a macro to be expanded, do it.
817 if (MacroDefinition MD = getMacroDefinition(&II)) {
818 auto *MI = MD.getMacroInfo();
819 assert(MI && "macro definition with no macro info?");
820 if (!DisableMacroExpansion) {
821 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
822 // C99 6.10.3p10: If the preprocessing token immediately after the
823 // macro name isn't a '(', this macro should not be expanded.
824 if (!MI->isFunctionLike() || isNextPPTokenLParen())
825 return HandleMacroExpandedIdentifier(Identifier, MD);
826 } else {
827 // C99 6.10.3.4p2 says that a disabled macro may never again be
828 // expanded, even if it's in a context where it could be expanded in the
829 // future.
831 if (MI->isObjectLike() || isNextPPTokenLParen())
832 Diag(Identifier, diag::pp_disabled_macro_expansion);
833 }
834 }
835 }
836
837 // If this identifier is a keyword in a newer Standard or proposed Standard,
838 // produce a warning. Don't warn if we're not considering macro expansion,
839 // since this identifier might be the name of a macro.
840 // FIXME: This warning is disabled in cases where it shouldn't be, like
841 // "#define constexpr constexpr", "int constexpr;"
842 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
843 Diag(Identifier, getIdentifierTable().getFutureCompatDiagKind(II, getLangOpts()))
844 << II.getName();
845 // Don't diagnose this keyword again in this translation unit.
846 II.setIsFutureCompatKeyword(false);
847 }
848
849 // If this is an extension token, diagnose its use.
850 // We avoid diagnosing tokens that originate from macro definitions.
851 // FIXME: This warning is disabled in cases where it shouldn't be,
852 // like "#define TY typeof", "TY(1) x".
853 if (II.isExtensionToken() && !DisableMacroExpansion)
854 Diag(Identifier, diag::ext_token_used);
855
856 // If this is the 'import' contextual keyword following an '@', note
857 // that the next token indicates a module name.
858 //
859 // Note that we do not treat 'import' as a contextual
860 // keyword when we're in a caching lexer, because caching lexers only get
861 // used in contexts where import declarations are disallowed.
862 //
863 // Likewise if this is the standard C++ import keyword.
864 if (((LastTokenWasAt && II.isModulesImport()) ||
865 Identifier.is(tok::kw_import)) &&
866 !InMacroArgs && !DisableMacroExpansion &&
867 (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
868 CurLexerKind != CLK_CachingLexer) {
869 ModuleImportLoc = Identifier.getLocation();
870 NamedModuleImportPath.clear();
871 IsAtImport = true;
872 ModuleImportExpectsIdentifier = true;
873 CurLexerKind = CLK_LexAfterModuleImport;
874 }
875 return true;
876}
877
879 ++LexLevel;
880
881 // We loop here until a lex function returns a token; this avoids recursion.
882 bool ReturnedToken;
883 do {
884 switch (CurLexerKind) {
885 case CLK_Lexer:
886 ReturnedToken = CurLexer->Lex(Result);
887 break;
888 case CLK_TokenLexer:
889 ReturnedToken = CurTokenLexer->Lex(Result);
890 break;
891 case CLK_CachingLexer:
892 CachingLex(Result);
893 ReturnedToken = true;
894 break;
895 case CLK_DependencyDirectivesLexer:
896 ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result);
897 break;
898 case CLK_LexAfterModuleImport:
899 ReturnedToken = LexAfterModuleImport(Result);
900 break;
901 }
902 } while (!ReturnedToken);
903
904 if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure)
905 return;
906
907 if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) {
908 // Remember the identifier before code completion token.
909 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
910 setCodeCompletionTokenRange(Result.getLocation(), Result.getEndLoc());
911 // Set IdenfitierInfo to null to avoid confusing code that handles both
912 // identifiers and completion tokens.
913 Result.setIdentifierInfo(nullptr);
914 }
915
916 // Update StdCXXImportSeqState to track our position within a C++20 import-seq
917 // if this token is being produced as a result of phase 4 of translation.
918 // Update TrackGMFState to decide if we are currently in a Global Module
919 // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state
920 // depends on the prevailing StdCXXImportSeq state in two cases.
921 if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
922 !Result.getFlag(Token::IsReinjected)) {
923 switch (Result.getKind()) {
924 case tok::l_paren: case tok::l_square: case tok::l_brace:
925 StdCXXImportSeqState.handleOpenBracket();
926 break;
927 case tok::r_paren: case tok::r_square:
928 StdCXXImportSeqState.handleCloseBracket();
929 break;
930 case tok::r_brace:
931 StdCXXImportSeqState.handleCloseBrace();
932 break;
933 // This token is injected to represent the translation of '#include "a.h"'
934 // into "import a.h;". Mimic the notional ';'.
935 case tok::annot_module_include:
936 case tok::semi:
937 TrackGMFState.handleSemi();
938 StdCXXImportSeqState.handleSemi();
939 ModuleDeclState.handleSemi();
940 break;
941 case tok::header_name:
942 case tok::annot_header_unit:
943 StdCXXImportSeqState.handleHeaderName();
944 break;
945 case tok::kw_export:
946 TrackGMFState.handleExport();
947 StdCXXImportSeqState.handleExport();
948 ModuleDeclState.handleExport();
949 break;
950 case tok::colon:
951 ModuleDeclState.handleColon();
952 break;
953 case tok::period:
954 ModuleDeclState.handlePeriod();
955 break;
956 case tok::identifier:
957 if (Result.getIdentifierInfo()->isModulesImport()) {
958 TrackGMFState.handleImport(StdCXXImportSeqState.afterTopLevelSeq());
959 StdCXXImportSeqState.handleImport();
960 if (StdCXXImportSeqState.afterImportSeq()) {
961 ModuleImportLoc = Result.getLocation();
962 NamedModuleImportPath.clear();
963 IsAtImport = false;
964 ModuleImportExpectsIdentifier = true;
965 CurLexerKind = CLK_LexAfterModuleImport;
966 }
967 break;
968 } else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) {
969 TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
970 ModuleDeclState.handleModule();
971 break;
972 } else {
973 ModuleDeclState.handleIdentifier(Result.getIdentifierInfo());
974 if (ModuleDeclState.isModuleCandidate())
975 break;
976 }
977 [[fallthrough]];
978 default:
979 TrackGMFState.handleMisc();
980 StdCXXImportSeqState.handleMisc();
981 ModuleDeclState.handleMisc();
982 break;
983 }
984 }
985
986 LastTokenWasAt = Result.is(tok::at);
987 --LexLevel;
988
989 if ((LexLevel == 0 || PreprocessToken) &&
990 !Result.getFlag(Token::IsReinjected)) {
991 if (LexLevel == 0)
992 ++TokenCount;
993 if (OnToken)
994 OnToken(Result);
995 }
996}
997
998/// Lex a header-name token (including one formed from header-name-tokens if
999/// \p AllowConcatenation is \c true).
1000///
1001/// \param FilenameTok Filled in with the next token. On success, this will
1002/// be either a header_name token. On failure, it will be whatever other
1003/// token was found instead.
1004/// \param AllowMacroExpansion If \c true, allow the header name to be formed
1005/// by macro expansion (concatenating tokens as necessary if the first
1006/// token is a '<').
1007/// \return \c true if we reached EOD or EOF while looking for a > token in
1008/// a concatenated header name and diagnosed it. \c false otherwise.
1009bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
1010 // Lex using header-name tokenization rules if tokens are being lexed from
1011 // a file. Just grab a token normally if we're in a macro expansion.
1012 if (CurPPLexer)
1013 CurPPLexer->LexIncludeFilename(FilenameTok);
1014 else
1015 Lex(FilenameTok);
1016
1017 // This could be a <foo/bar.h> file coming from a macro expansion. In this
1018 // case, glue the tokens together into an angle_string_literal token.
1019 SmallString<128> FilenameBuffer;
1020 if (FilenameTok.is(tok::less) && AllowMacroExpansion) {
1021 bool StartOfLine = FilenameTok.isAtStartOfLine();
1022 bool LeadingSpace = FilenameTok.hasLeadingSpace();
1023 bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1024
1025 SourceLocation Start = FilenameTok.getLocation();
1026 SourceLocation End;
1027 FilenameBuffer.push_back('<');
1028
1029 // Consume tokens until we find a '>'.
1030 // FIXME: A header-name could be formed starting or ending with an
1031 // alternative token. It's not clear whether that's ill-formed in all
1032 // cases.
1033 while (FilenameTok.isNot(tok::greater)) {
1034 Lex(FilenameTok);
1035 if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
1036 Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
1037 Diag(Start, diag::note_matching) << tok::less;
1038 return true;
1039 }
1040
1041 End = FilenameTok.getLocation();
1042
1043 // FIXME: Provide code completion for #includes.
1044 if (FilenameTok.is(tok::code_completion)) {
1046 Lex(FilenameTok);
1047 continue;
1048 }
1049
1050 // Append the spelling of this token to the buffer. If there was a space
1051 // before it, add it now.
1052 if (FilenameTok.hasLeadingSpace())
1053 FilenameBuffer.push_back(' ');
1054
1055 // Get the spelling of the token, directly into FilenameBuffer if
1056 // possible.
1057 size_t PreAppendSize = FilenameBuffer.size();
1058 FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
1059
1060 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1061 unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
1062
1063 // If the token was spelled somewhere else, copy it into FilenameBuffer.
1064 if (BufPtr != &FilenameBuffer[PreAppendSize])
1065 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1066
1067 // Resize FilenameBuffer to the correct size.
1068 if (FilenameTok.getLength() != ActualLen)
1069 FilenameBuffer.resize(PreAppendSize + ActualLen);
1070 }
1071
1072 FilenameTok.startToken();
1073 FilenameTok.setKind(tok::header_name);
1074 FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine);
1075 FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace);
1076 FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro);
1077 CreateString(FilenameBuffer, FilenameTok, Start, End);
1078 } else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) {
1079 // Convert a string-literal token of the form " h-char-sequence "
1080 // (produced by macro expansion) into a header-name token.
1081 //
1082 // The rules for header-names don't quite match the rules for
1083 // string-literals, but all the places where they differ result in
1084 // undefined behavior, so we can and do treat them the same.
1085 //
1086 // A string-literal with a prefix or suffix is not translated into a
1087 // header-name. This could theoretically be observable via the C++20
1088 // context-sensitive header-name formation rules.
1089 StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
1090 if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1091 FilenameTok.setKind(tok::header_name);
1092 }
1093
1094 return false;
1095}
1096
1097/// Collect the tokens of a C++20 pp-import-suffix.
1099 // FIXME: For error recovery, consider recognizing attribute syntax here
1100 // and terminating / diagnosing a missing semicolon if we find anything
1101 // else? (Can we leave that to the parser?)
1102 unsigned BracketDepth = 0;
1103 while (true) {
1104 Toks.emplace_back();
1105 Lex(Toks.back());
1106
1107 switch (Toks.back().getKind()) {
1108 case tok::l_paren: case tok::l_square: case tok::l_brace:
1109 ++BracketDepth;
1110 break;
1111
1112 case tok::r_paren: case tok::r_square: case tok::r_brace:
1113 if (BracketDepth == 0)
1114 return;
1115 --BracketDepth;
1116 break;
1117
1118 case tok::semi:
1119 if (BracketDepth == 0)
1120 return;
1121 break;
1122
1123 case tok::eof:
1124 return;
1125
1126 default:
1127 break;
1128 }
1129 }
1130}
1131
1132
1133/// Lex a token following the 'import' contextual keyword.
1134///
1135/// pp-import: [C++20]
1136/// import header-name pp-import-suffix[opt] ;
1137/// import header-name-tokens pp-import-suffix[opt] ;
1138/// [ObjC] @ import module-name ;
1139/// [Clang] import module-name ;
1140///
1141/// header-name-tokens:
1142/// string-literal
1143/// < [any sequence of preprocessing-tokens other than >] >
1144///
1145/// module-name:
1146/// module-name-qualifier[opt] identifier
1147///
1148/// module-name-qualifier
1149/// module-name-qualifier[opt] identifier .
1150///
1151/// We respond to a pp-import by importing macros from the named module.
1153 // Figure out what kind of lexer we actually have.
1155
1156 // Lex the next token. The header-name lexing rules are used at the start of
1157 // a pp-import.
1158 //
1159 // For now, we only support header-name imports in C++20 mode.
1160 // FIXME: Should we allow this in all language modes that support an import
1161 // declaration as an extension?
1162 if (NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1163 if (LexHeaderName(Result))
1164 return true;
1165
1166 if (Result.is(tok::colon) && ModuleDeclState.isNamedModule()) {
1167 std::string Name = ModuleDeclState.getPrimaryName().str();
1168 Name += ":";
1169 NamedModuleImportPath.push_back(
1170 {getIdentifierInfo(Name), Result.getLocation()});
1171 CurLexerKind = CLK_LexAfterModuleImport;
1172 return true;
1173 }
1174 } else {
1175 Lex(Result);
1176 }
1177
1178 // Allocate a holding buffer for a sequence of tokens and introduce it into
1179 // the token stream.
1180 auto EnterTokens = [this](ArrayRef<Token> Toks) {
1181 auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1182 std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1183 EnterTokenStream(std::move(ToksCopy), Toks.size(),
1184 /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1185 };
1186
1187 bool ImportingHeader = Result.is(tok::header_name);
1188 // Check for a header-name.
1190 if (ImportingHeader) {
1191 // Enter the header-name token into the token stream; a Lex action cannot
1192 // both return a token and cache tokens (doing so would corrupt the token
1193 // cache if the call to Lex comes from CachingLex / PeekAhead).
1194 Suffix.push_back(Result);
1195
1196 // Consume the pp-import-suffix and expand any macros in it now. We'll add
1197 // it back into the token stream later.
1198 CollectPpImportSuffix(Suffix);
1199 if (Suffix.back().isNot(tok::semi)) {
1200 // This is not a pp-import after all.
1201 EnterTokens(Suffix);
1202 return false;
1203 }
1204
1205 // C++2a [cpp.module]p1:
1206 // The ';' preprocessing-token terminating a pp-import shall not have
1207 // been produced by macro replacement.
1208 SourceLocation SemiLoc = Suffix.back().getLocation();
1209 if (SemiLoc.isMacroID())
1210 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1211
1212 // Reconstitute the import token.
1213 Token ImportTok;
1214 ImportTok.startToken();
1215 ImportTok.setKind(tok::kw_import);
1216 ImportTok.setLocation(ModuleImportLoc);
1217 ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
1218 ImportTok.setLength(6);
1219
1220 auto Action = HandleHeaderIncludeOrImport(
1221 /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
1222 switch (Action.Kind) {
1223 case ImportAction::None:
1224 break;
1225
1226 case ImportAction::ModuleBegin:
1227 // Let the parser know we're textually entering the module.
1228 Suffix.emplace_back();
1229 Suffix.back().startToken();
1230 Suffix.back().setKind(tok::annot_module_begin);
1231 Suffix.back().setLocation(SemiLoc);
1232 Suffix.back().setAnnotationEndLoc(SemiLoc);
1233 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1234 [[fallthrough]];
1235
1236 case ImportAction::ModuleImport:
1237 case ImportAction::HeaderUnitImport:
1238 case ImportAction::SkippedModuleImport:
1239 // We chose to import (or textually enter) the file. Convert the
1240 // header-name token into a header unit annotation token.
1241 Suffix[0].setKind(tok::annot_header_unit);
1242 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1243 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1244 // FIXME: Call the moduleImport callback?
1245 break;
1246 case ImportAction::Failure:
1247 assert(TheModuleLoader.HadFatalFailure &&
1248 "This should be an early exit only to a fatal error");
1249 Result.setKind(tok::eof);
1250 CurLexer->cutOffLexing();
1251 EnterTokens(Suffix);
1252 return true;
1253 }
1254
1255 EnterTokens(Suffix);
1256 return false;
1257 }
1258
1259 // The token sequence
1260 //
1261 // import identifier (. identifier)*
1262 //
1263 // indicates a module import directive. We already saw the 'import'
1264 // contextual keyword, so now we're looking for the identifiers.
1265 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1266 // We expected to see an identifier here, and we did; continue handling
1267 // identifiers.
1268 NamedModuleImportPath.push_back(
1269 std::make_pair(Result.getIdentifierInfo(), Result.getLocation()));
1270 ModuleImportExpectsIdentifier = false;
1271 CurLexerKind = CLK_LexAfterModuleImport;
1272 return true;
1273 }
1274
1275 // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1276 // see the next identifier. (We can also see a '[[' that begins an
1277 // attribute-specifier-seq here under the Standard C++ Modules.)
1278 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1279 ModuleImportExpectsIdentifier = true;
1280 CurLexerKind = CLK_LexAfterModuleImport;
1281 return true;
1282 }
1283
1284 // If we didn't recognize a module name at all, this is not a (valid) import.
1285 if (NamedModuleImportPath.empty() || Result.is(tok::eof))
1286 return true;
1287
1288 // Consume the pp-import-suffix and expand any macros in it now, if we're not
1289 // at the semicolon already.
1290 SourceLocation SemiLoc = Result.getLocation();
1291 if (Result.isNot(tok::semi)) {
1292 Suffix.push_back(Result);
1293 CollectPpImportSuffix(Suffix);
1294 if (Suffix.back().isNot(tok::semi)) {
1295 // This is not an import after all.
1296 EnterTokens(Suffix);
1297 return false;
1298 }
1299 SemiLoc = Suffix.back().getLocation();
1300 }
1301
1302 // Under the standard C++ Modules, the dot is just part of the module name,
1303 // and not a real hierarchy separator. Flatten such module names now.
1304 //
1305 // FIXME: Is this the right level to be performing this transformation?
1306 std::string FlatModuleName;
1307 if (getLangOpts().CPlusPlusModules) {
1308 for (auto &Piece : NamedModuleImportPath) {
1309 // If the FlatModuleName ends with colon, it implies it is a partition.
1310 if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
1311 FlatModuleName += ".";
1312 FlatModuleName += Piece.first->getName();
1313 }
1314 SourceLocation FirstPathLoc = NamedModuleImportPath[0].second;
1315 NamedModuleImportPath.clear();
1316 NamedModuleImportPath.push_back(
1317 std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1318 }
1319
1320 Module *Imported = nullptr;
1321 // We don't/shouldn't load the standard c++20 modules when preprocessing.
1322 if (getLangOpts().Modules && !isInImportingCXXNamedModules()) {
1323 Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1324 NamedModuleImportPath,
1326 /*IsInclusionDirective=*/false);
1327 if (Imported)
1328 makeModuleVisible(Imported, SemiLoc);
1329 }
1330
1331 if (Callbacks)
1332 Callbacks->moduleImport(ModuleImportLoc, NamedModuleImportPath, Imported);
1333
1334 if (!Suffix.empty()) {
1335 EnterTokens(Suffix);
1336 return false;
1337 }
1338 return true;
1339}
1340
1342 CurSubmoduleState->VisibleModules.setVisible(
1343 M, Loc, [](Module *) {},
1344 [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1345 // FIXME: Include the path in the diagnostic.
1346 // FIXME: Include the import location for the conflicting module.
1347 Diag(ModuleImportLoc, diag::warn_module_conflict)
1348 << Path[0]->getFullModuleName()
1349 << Conflict->getFullModuleName()
1350 << Message;
1351 });
1352
1353 // Add this module to the imports list of the currently-built submodule.
1354 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1355 BuildingSubmoduleStack.back().M->Imports.insert(M);
1356}
1357
1359 const char *DiagnosticTag,
1360 bool AllowMacroExpansion) {
1361 // We need at least one string literal.
1362 if (Result.isNot(tok::string_literal)) {
1363 Diag(Result, diag::err_expected_string_literal)
1364 << /*Source='in...'*/0 << DiagnosticTag;
1365 return false;
1366 }
1367
1368 // Lex string literal tokens, optionally with macro expansion.
1369 SmallVector<Token, 4> StrToks;
1370 do {
1371 StrToks.push_back(Result);
1372
1373 if (Result.hasUDSuffix())
1374 Diag(Result, diag::err_invalid_string_udl);
1375
1376 if (AllowMacroExpansion)
1377 Lex(Result);
1378 else
1380 } while (Result.is(tok::string_literal));
1381
1382 // Concatenate and parse the strings.
1383 StringLiteralParser Literal(StrToks, *this);
1384 assert(Literal.isOrdinary() && "Didn't allow wide strings in");
1385
1386 if (Literal.hadError)
1387 return false;
1388
1389 if (Literal.Pascal) {
1390 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1391 << /*Source='in...'*/0 << DiagnosticTag;
1392 return false;
1393 }
1394
1395 String = std::string(Literal.GetString());
1396 return true;
1397}
1398
1400 assert(Tok.is(tok::numeric_constant));
1401 SmallString<8> IntegerBuffer;
1402 bool NumberInvalid = false;
1403 StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1404 if (NumberInvalid)
1405 return false;
1406 NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
1408 getDiagnostics());
1409 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1410 return false;
1411 llvm::APInt APVal(64, 0);
1412 if (Literal.GetIntegerValue(APVal))
1413 return false;
1414 Lex(Tok);
1415 Value = APVal.getLimitedValue();
1416 return true;
1417}
1418
1420 assert(Handler && "NULL comment handler");
1421 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1422 "Comment handler already registered");
1423 CommentHandlers.push_back(Handler);
1424}
1425
1427 std::vector<CommentHandler *>::iterator Pos =
1428 llvm::find(CommentHandlers, Handler);
1429 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1430 CommentHandlers.erase(Pos);
1431}
1432
1434 bool AnyPendingTokens = false;
1435 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1436 HEnd = CommentHandlers.end();
1437 H != HEnd; ++H) {
1438 if ((*H)->HandleComment(*this, Comment))
1439 AnyPendingTokens = true;
1440 }
1441 if (!AnyPendingTokens || getCommentRetentionState())
1442 return false;
1443 Lex(result);
1444 return true;
1445}
1446
1447void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1448 const MacroAnnotations &A =
1449 getMacroAnnotations(Identifier.getIdentifierInfo());
1450 assert(A.DeprecationInfo &&
1451 "Macro deprecation warning without recorded annotation!");
1452 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1453 if (Info.Message.empty())
1454 Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1455 << Identifier.getIdentifierInfo() << 0;
1456 else
1457 Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1458 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1459 Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1460}
1461
1462void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1463 const MacroAnnotations &A =
1464 getMacroAnnotations(Identifier.getIdentifierInfo());
1465 assert(A.RestrictExpansionInfo &&
1466 "Macro restricted expansion warning without recorded annotation!");
1467 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1468 if (Info.Message.empty())
1469 Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1470 << Identifier.getIdentifierInfo() << 0;
1471 else
1472 Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1473 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1474 Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1475}
1476
1477void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1478 bool IsUndef) const {
1479 const MacroAnnotations &A =
1480 getMacroAnnotations(Identifier.getIdentifierInfo());
1481 assert(A.FinalAnnotationLoc &&
1482 "Final macro warning without recorded annotation!");
1483
1484 Diag(Identifier, diag::warn_pragma_final_macro)
1485 << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1486 Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1487}
1488
1490 const SourceLocation &Loc) const {
1491 // Try to find a region in `SafeBufferOptOutMap` where `Loc` is in:
1492 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1493 SafeBufferOptOutMap,
1494 [&SourceMgr,
1495 &Loc](const std::pair<SourceLocation, SourceLocation> &Region) {
1496 return SourceMgr.isBeforeInTranslationUnit(Region.second, Loc);
1497 });
1498
1499 if (FirstRegionEndingAfterLoc != SafeBufferOptOutMap.end()) {
1500 // To test if the start location of the found region precedes `Loc`:
1501 return SourceMgr.isBeforeInTranslationUnit(FirstRegionEndingAfterLoc->first,
1502 Loc);
1503 }
1504 // If we do not find a region whose end location passes `Loc`, we want to
1505 // check if the current region is still open:
1506 if (!SafeBufferOptOutMap.empty() &&
1507 SafeBufferOptOutMap.back().first == SafeBufferOptOutMap.back().second)
1508 return SourceMgr.isBeforeInTranslationUnit(SafeBufferOptOutMap.back().first,
1509 Loc);
1510 return false;
1511}
1512
1514 bool isEnter, const SourceLocation &Loc) {
1515 if (isEnter) {
1517 return true; // invalid enter action
1518 InSafeBufferOptOutRegion = true;
1519 CurrentSafeBufferOptOutStart = Loc;
1520
1521 // To set the start location of a new region:
1522
1523 if (!SafeBufferOptOutMap.empty()) {
1524 [[maybe_unused]] auto *PrevRegion = &SafeBufferOptOutMap.back();
1525 assert(PrevRegion->first != PrevRegion->second &&
1526 "Shall not begin a safe buffer opt-out region before closing the "
1527 "previous one.");
1528 }
1529 // If the start location equals to the end location, we call the region a
1530 // open region or a unclosed region (i.e., end location has not been set
1531 // yet).
1532 SafeBufferOptOutMap.emplace_back(Loc, Loc);
1533 } else {
1535 return true; // invalid enter action
1536 InSafeBufferOptOutRegion = false;
1537
1538 // To set the end location of the current open region:
1539
1540 assert(!SafeBufferOptOutMap.empty() &&
1541 "Misordered safe buffer opt-out regions");
1542 auto *CurrRegion = &SafeBufferOptOutMap.back();
1543 assert(CurrRegion->first == CurrRegion->second &&
1544 "Set end location to a closed safe buffer opt-out region");
1545 CurrRegion->second = Loc;
1546 }
1547 return false;
1548}
1549
1551 return InSafeBufferOptOutRegion;
1552}
1554 StartLoc = CurrentSafeBufferOptOutStart;
1555 return InSafeBufferOptOutRegion;
1556}
1557
1558ModuleLoader::~ModuleLoader() = default;
1559
1561
1563
1565
1567 if (Record)
1568 return;
1569
1570 Record = new PreprocessingRecord(getSourceManager());
1571 addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1572}
#define SM(sm)
Definition: Cuda.cpp:80
Defines enum values for all the target-independent builtin functions.
Defines the clang::FileManager interface and associated types.
Defines the FileSystemStatCache interface.
StringRef Identifier
Definition: Format.cpp:2804
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.
Defines the clang::LangOptions interface.
Defines the clang::MacroInfo and clang::MacroDirective classes.
Defines the clang::Module class, which describes a module in the source code.
Defines the PreprocessorLexer interface.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
Defines the clang::Preprocessor interface.
Defines the clang::SourceLocation class and associated facilities.
Defines the SourceManager interface.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive.
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Abstract base class that describes a handler that will receive source ranges for each of the comments...
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:192
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
Cached information about one file (either on disk or in the virtual file system).
Definition: FileEntry.h:353
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isValid() const
bool isInvalid() const
Encapsulates the information needed to find the file referenced by a #include or #include_next,...
Definition: HeaderSearch.h:223
Module * lookupModule(StringRef ModuleName, SourceLocation ImportLoc=SourceLocation(), bool AllowSearch=true, bool AllowExtraModuleMapSearch=false)
Lookup a module Search for a module with the given name.
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Provides lookups to, and iteration over, IdentiferInfo objects.
One of these records is kept for each identifier that is lexed.
bool isModulesImport() const
Determine whether this is the contextual keyword import.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setIsFutureCompatKeyword(bool Val)
StringRef getName() const
Return the actual identifier string.
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
bool isExtensionToken() const
get/setExtension - Initialize information about whether or not this language token is an extension.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:82
@ FEM_UnsetOnCommandLine
Used only for FE option processing; this is only used to indicate that the user did not specify an ex...
Definition: LangOptions.h:296
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:30
A description of the current definition of a macro.
Definition: MacroInfo.h:587
SourceLocation getLocation() const
Definition: MacroInfo.h:485
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
const_tokens_iterator tokens_begin() const
Definition: MacroInfo.h:244
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:235
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:237
bool isObjectLike() const
Definition: MacroInfo.h:202
Abstract interface for a module loader.
Definition: ModuleLoader.h:82
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
virtual ~ModuleLoader()
Represents a macro directive exported by a module.
Definition: MacroInfo.h:511
Describes a module or submodule.
Definition: Module.h:98
@ Hidden
All of the names in this module are hidden.
Definition: Module.h:361
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber,...
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:96
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
void FinalizeForModelFile()
Cleanup after model file parsing.
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
bool markIncluded(const FileEntry *File)
Mark the file as included.
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isSafeBufferOptOut(const SourceManager &SourceMgr, const SourceLocation &Loc) const
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
void DumpMacro(const MacroInfo &MI) const
void setCodeCompletionReached()
Note that we hit the code-completion point.
void makeModuleVisible(Module *M, SourceLocation Loc)
bool isInImportingCXXNamedModules() const
If we're importing a standard C++20 Named Modules.
void Lex(Token &Result)
Lex the next token for this preprocessor.
const TranslationUnitKind TUKind
The kind of translation unit we are processing.
Definition: Preprocessor.h:282
bool EnterSourceFile(FileID FID, ConstSearchDirIterator Dir, SourceLocation Loc, bool IsFirstIncludeOfFile=true)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
bool enterOrExitSafeBufferOptOutRegion(bool isEnter, const SourceLocation &Loc)
Alter the state of whether this PP currently is in a "-Wunsafe-buffer-usage" opt-out region.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc.
const MacroAnnotations & getMacroAnnotations(const IdentifierInfo *II) const
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
macro_iterator macro_end(bool IncludeExternalMacros=true) const
SourceManager & getSourceManager() const
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
bool getCommentRetentionState() const
Module * getCurrentModuleImplementation()
Retrieves the module whose implementation we're current compiling, if any.
MacroMap::const_iterator macro_iterator
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions,...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
bool isPPInSafeBufferOptOutRegion()
void setCurrentFPEvalMethod(SourceLocation PragmaLoc, LangOptions::FPEvalMethodKind Val)
const TargetInfo & getTargetInfo() const
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void DumpLocation(SourceLocation Loc) const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleComment(Token &result, SourceRange Comment)
HeaderSearch & getHeaderSearchInfo() const
ExternalPreprocessorSource * getExternalSource() const
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
OptionalFileEntryRef LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, ConstSearchDirIterator FromDir, const FileEntry *FromFile, ConstSearchDirIterator *CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false, bool OpenFile=true, bool CacheFailures=true)
Given a "foo" or <foo> reference, look up the indicated file.
IdentifierTable & getIdentifierTable()
const LangOptions & getLangOpts() const
void setTUFPEvalMethod(LangOptions::FPEvalMethodKind Val)
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void PoisonSEHIdentifiers(bool Poison=true)
size_t getTotalMemory() const
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
DiagnosticsEngine & getDiagnostics() const
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop.
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens.
Definition: ScratchBuffer.h:24
Encodes a location in the source.
bool isValid() const
Return true if this is a valid SourceLocation object.
void print(raw_ostream &OS, const SourceManager &SM) const
SourceLocation getLocWithOffset(IntTy Offset) const
Return a source location with the specified offset from this SourceLocation.
This class handles loading and caching of source files into memory.
void overrideFileContents(const FileEntry *SourceFile, const llvm::MemoryBufferRef &Buffer)
Override the contents of the given source file by providing an already-allocated buffer.
FileID getMainFileID() const
Returns the FileID of the main source file.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer.
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
std::optional< llvm::MemoryBufferRef > getMemoryBufferForFileOrNone(const FileEntry *File)
Retrieve the memory buffer associated with the given file.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
SourceLocation createExpansionLoc(SourceLocation SpellingLoc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned Length, bool ExpansionIsTokenRange=true, int LoadedID=0, SourceLocation::UIntTy LoadedOffset=0)
Creates an expansion SLocEntry for a macro use.
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
A trivial tuple used to represent a source range.
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
Exposes information about the current target.
Definition: TargetInfo.h:206
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:186
void setLiteralData(const char *Ptr)
Definition: Token.h:228
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:303
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:115
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file.
Definition: Token.h:131
unsigned getLength() const
Definition: Token.h:134
void setLength(unsigned Len)
Definition: Token.h:140
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6....
Definition: Token.h:283
void setKind(tok::TokenKind K)
Definition: Token.h:94
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....
Definition: Token.h:98
tok::TokenKind getKind() const
Definition: Token.h:93
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:275
@ DisableExpand
Definition: Token.h:78
@ IsReinjected
Definition: Token.h:88
@ LeadingEmptyMacro
Definition: Token.h:80
@ LeadingSpace
Definition: Token.h:76
@ StartOfLine
Definition: Token.h:74
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:279
void setLocation(SourceLocation L)
Definition: Token.h:139
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
Definition: Token.h:296
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:216
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
Definition: Token.h:100
bool isNot(tok::TokenKind K) const
Definition: Token.h:99
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
void startToken()
Reset all flags to cleared.
Definition: Token.h:176
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:292
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:195
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Definition: Token.h:266
Defines the clang::TargetInfo interface.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:24
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
@ Result
The result type of a method or function.
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:917
@ TU_Prefix
The translation unit is a prefix to a translation unit, and is not complete.
Definition: LangOptions.h:923
Definition: Format.h:4756
#define true
Definition: stdbool.h:21