TokenLexer.cpp File Reference

#include "clang/Lex/TokenLexer.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/IdentifierTable.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Lex/LexDiagnostic.h"
#include "clang/Lex/Lexer.h"
#include "clang/Lex/MacroArgs.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/Token.h"
#include "clang/Lex/VariadicMacroSupport.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/iterator_range.h"
#include <cassert>
#include <cstring>
#include <optional>

Go to the source code of this file.

Functions
static bool	isWideStringLiteralFromMacro (const Token &FirstTok, const Token &SecondTok)
	Checks if two tokens form wide string literal.
static void	updateConsecutiveMacroArgTokens (SourceManager &SM, SourceLocation ExpandLoc, Token *&begin_tokens, Token *end_tokens)
	Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.

Function Documentation

isWideStringLiteralFromMacro()

static bool isWideStringLiteralFromMacro ( const Token &  FirstTok, const Token &  SecondTok  )

static

Checks if two tokens form wide string literal.

Definition at line 614 of file TokenLexer.cpp.

References clang::Token::getIdentifierInfo(), clang::Token::is(), clang::Token::isLiteral(), clang::IdentifierInfo::isStr(), and clang::Token::stringifiedInMacro().

Referenced by clang::TokenLexer::Lex().

updateConsecutiveMacroArgTokens()

static void updateConsecutiveMacroArgTokens ( SourceManager &  SM, SourceLocation  ExpandLoc, Token *&  begin_tokens, Token *  end_tokens  )

static

Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.

e.g for assert(foo == bar); There will be a single SLocEntry for the "foo == bar" chunk and locations for the 'foo', '==', 'bar' tokens will point inside that chunk.

• begin_tokens will be updated to a position past all the found consecutive tokens.

Definition at line 987 of file TokenLexer.cpp.

References clang::All, clang::Token::getLocation(), clang::SourceLocation::getLocWithOffset(), clang::SourceLocation::getRawEncoding(), clang::SourceLocation::isFileID(), clang::Last, SM, and clang::T.