clang  10.0.0svn
LiteralSupport.cpp
Go to the documentation of this file.
1 //===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the NumericLiteralParser, CharLiteralParser, and
10 // StringLiteralParser interfaces.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "clang/Basic/CharInfo.h"
18 #include "clang/Basic/TargetInfo.h"
20 #include "clang/Lex/Lexer.h"
21 #include "clang/Lex/Preprocessor.h"
22 #include "clang/Lex/Token.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/StringExtras.h"
26 #include "llvm/ADT/StringSwitch.h"
27 #include "llvm/Support/ConvertUTF.h"
28 #include "llvm/Support/ErrorHandling.h"
29 #include <algorithm>
30 #include <cassert>
31 #include <cstddef>
32 #include <cstdint>
33 #include <cstring>
34 #include <string>
35 
36 using namespace clang;
37 
38 static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
39  switch (kind) {
40  default: llvm_unreachable("Unknown token type!");
41  case tok::char_constant:
42  case tok::string_literal:
43  case tok::utf8_char_constant:
44  case tok::utf8_string_literal:
45  return Target.getCharWidth();
46  case tok::wide_char_constant:
47  case tok::wide_string_literal:
48  return Target.getWCharWidth();
49  case tok::utf16_char_constant:
50  case tok::utf16_string_literal:
51  return Target.getChar16Width();
52  case tok::utf32_char_constant:
53  case tok::utf32_string_literal:
54  return Target.getChar32Width();
55  }
56 }
57 
59  FullSourceLoc TokLoc,
60  const char *TokBegin,
61  const char *TokRangeBegin,
62  const char *TokRangeEnd) {
64  Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin,
65  TokLoc.getManager(), Features);
67  Lexer::AdvanceToTokenCharacter(Begin, TokRangeEnd - TokRangeBegin,
68  TokLoc.getManager(), Features);
69  return CharSourceRange::getCharRange(Begin, End);
70 }
71 
72 /// Produce a diagnostic highlighting some portion of a literal.
73 ///
74 /// Emits the diagnostic \p DiagID, highlighting the range of characters from
75 /// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be
76 /// a substring of a spelling buffer for the token beginning at \p TokBegin.
78  const LangOptions &Features, FullSourceLoc TokLoc,
79  const char *TokBegin, const char *TokRangeBegin,
80  const char *TokRangeEnd, unsigned DiagID) {
82  Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin,
83  TokLoc.getManager(), Features);
84  return Diags->Report(Begin, DiagID) <<
85  MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd);
86 }
87 
88 /// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
89 /// either a character or a string literal.
90 static unsigned ProcessCharEscape(const char *ThisTokBegin,
91  const char *&ThisTokBuf,
92  const char *ThisTokEnd, bool &HadError,
93  FullSourceLoc Loc, unsigned CharWidth,
94  DiagnosticsEngine *Diags,
95  const LangOptions &Features) {
96  const char *EscapeBegin = ThisTokBuf;
97 
98  // Skip the '\' char.
99  ++ThisTokBuf;
100 
101  // We know that this character can't be off the end of the buffer, because
102  // that would have been \", which would not have been the end of string.
103  unsigned ResultChar = *ThisTokBuf++;
104  switch (ResultChar) {
105  // These map to themselves.
106  case '\\': case '\'': case '"': case '?': break;
107 
108  // These have fixed mappings.
109  case 'a':
110  // TODO: K&R: the meaning of '\\a' is different in traditional C
111  ResultChar = 7;
112  break;
113  case 'b':
114  ResultChar = 8;
115  break;
116  case 'e':
117  if (Diags)
118  Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
119  diag::ext_nonstandard_escape) << "e";
120  ResultChar = 27;
121  break;
122  case 'E':
123  if (Diags)
124  Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
125  diag::ext_nonstandard_escape) << "E";
126  ResultChar = 27;
127  break;
128  case 'f':
129  ResultChar = 12;
130  break;
131  case 'n':
132  ResultChar = 10;
133  break;
134  case 'r':
135  ResultChar = 13;
136  break;
137  case 't':
138  ResultChar = 9;
139  break;
140  case 'v':
141  ResultChar = 11;
142  break;
143  case 'x': { // Hex escape.
144  ResultChar = 0;
145  if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) {
146  if (Diags)
147  Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
148  diag::err_hex_escape_no_digits) << "x";
149  HadError = true;
150  break;
151  }
152 
153  // Hex escapes are a maximal series of hex digits.
154  bool Overflow = false;
155  for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
156  int CharVal = llvm::hexDigitValue(ThisTokBuf[0]);
157  if (CharVal == -1) break;
158  // About to shift out a digit?
159  if (ResultChar & 0xF0000000)
160  Overflow = true;
161  ResultChar <<= 4;
162  ResultChar |= CharVal;
163  }
164 
165  // See if any bits will be truncated when evaluated as a character.
166  if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
167  Overflow = true;
168  ResultChar &= ~0U >> (32-CharWidth);
169  }
170 
171  // Check for overflow.
172  if (Overflow && Diags) // Too many digits to fit in
173  Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
174  diag::err_escape_too_large) << 0;
175  break;
176  }
177  case '0': case '1': case '2': case '3':
178  case '4': case '5': case '6': case '7': {
179  // Octal escapes.
180  --ThisTokBuf;
181  ResultChar = 0;
182 
183  // Octal escapes are a series of octal digits with maximum length 3.
184  // "\0123" is a two digit sequence equal to "\012" "3".
185  unsigned NumDigits = 0;
186  do {
187  ResultChar <<= 3;
188  ResultChar |= *ThisTokBuf++ - '0';
189  ++NumDigits;
190  } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
191  ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
192 
193  // Check for overflow. Reject '\777', but not L'\777'.
194  if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
195  if (Diags)
196  Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
197  diag::err_escape_too_large) << 1;
198  ResultChar &= ~0U >> (32-CharWidth);
199  }
200  break;
201  }
202 
203  // Otherwise, these are not valid escapes.
204  case '(': case '{': case '[': case '%':
205  // GCC accepts these as extensions. We warn about them as such though.
206  if (Diags)
207  Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
208  diag::ext_nonstandard_escape)
209  << std::string(1, ResultChar);
210  break;
211  default:
212  if (!Diags)
213  break;
214 
215  if (isPrintable(ResultChar))
216  Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
217  diag::ext_unknown_escape)
218  << std::string(1, ResultChar);
219  else
220  Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
221  diag::ext_unknown_escape)
222  << "x" + llvm::utohexstr(ResultChar);
223  break;
224  }
225 
226  return ResultChar;
227 }
228 
229 static void appendCodePoint(unsigned Codepoint,
231  char ResultBuf[4];
232  char *ResultPtr = ResultBuf;
233  bool Res = llvm::ConvertCodePointToUTF8(Codepoint, ResultPtr);
234  (void)Res;
235  assert(Res && "Unexpected conversion failure");
236  Str.append(ResultBuf, ResultPtr);
237 }
238 
239 void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) {
240  for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; ++I) {
241  if (*I != '\\') {
242  Buf.push_back(*I);
243  continue;
244  }
245 
246  ++I;
247  assert(*I == 'u' || *I == 'U');
248 
249  unsigned NumHexDigits;
250  if (*I == 'u')
251  NumHexDigits = 4;
252  else
253  NumHexDigits = 8;
254 
255  assert(I + NumHexDigits <= E);
256 
257  uint32_t CodePoint = 0;
258  for (++I; NumHexDigits != 0; ++I, --NumHexDigits) {
259  unsigned Value = llvm::hexDigitValue(*I);
260  assert(Value != -1U);
261 
262  CodePoint <<= 4;
263  CodePoint += Value;
264  }
265 
266  appendCodePoint(CodePoint, Buf);
267  --I;
268  }
269 }
270 
271 /// ProcessUCNEscape - Read the Universal Character Name, check constraints and
272 /// return the UTF32.
273 static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
274  const char *ThisTokEnd,
275  uint32_t &UcnVal, unsigned short &UcnLen,
276  FullSourceLoc Loc, DiagnosticsEngine *Diags,
277  const LangOptions &Features,
278  bool in_char_string_literal = false) {
279  const char *UcnBegin = ThisTokBuf;
280 
281  // Skip the '\u' char's.
282  ThisTokBuf += 2;
283 
284  if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) {
285  if (Diags)
286  Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
287  diag::err_hex_escape_no_digits) << StringRef(&ThisTokBuf[-1], 1);
288  return false;
289  }
290  UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
291  unsigned short UcnLenSave = UcnLen;
292  for (; ThisTokBuf != ThisTokEnd && UcnLenSave; ++ThisTokBuf, UcnLenSave--) {
293  int CharVal = llvm::hexDigitValue(ThisTokBuf[0]);
294  if (CharVal == -1) break;
295  UcnVal <<= 4;
296  UcnVal |= CharVal;
297  }
298  // If we didn't consume the proper number of digits, there is a problem.
299  if (UcnLenSave) {
300  if (Diags)
301  Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
302  diag::err_ucn_escape_incomplete);
303  return false;
304  }
305 
306  // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2]
307  if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints
308  UcnVal > 0x10FFFF) { // maximum legal UTF32 value
309  if (Diags)
310  Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
311  diag::err_ucn_escape_invalid);
312  return false;
313  }
314 
315  // C++11 allows UCNs that refer to control characters and basic source
316  // characters inside character and string literals
317  if (UcnVal < 0xa0 &&
318  (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) { // $, @, `
319  bool IsError = (!Features.CPlusPlus11 || !in_char_string_literal);
320  if (Diags) {
321  char BasicSCSChar = UcnVal;
322  if (UcnVal >= 0x20 && UcnVal < 0x7f)
323  Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
324  IsError ? diag::err_ucn_escape_basic_scs :
325  diag::warn_cxx98_compat_literal_ucn_escape_basic_scs)
326  << StringRef(&BasicSCSChar, 1);
327  else
328  Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
329  IsError ? diag::err_ucn_control_character :
330  diag::warn_cxx98_compat_literal_ucn_control_character);
331  }
332  if (IsError)
333  return false;
334  }
335 
336  if (!Features.CPlusPlus && !Features.C99 && Diags)
337  Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
338  diag::warn_ucn_not_valid_in_c89_literal);
339 
340  return true;
341 }
342 
343 /// MeasureUCNEscape - Determine the number of bytes within the resulting string
344 /// which this UCN will occupy.
345 static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
346  const char *ThisTokEnd, unsigned CharByteWidth,
347  const LangOptions &Features, bool &HadError) {
348  // UTF-32: 4 bytes per escape.
349  if (CharByteWidth == 4)
350  return 4;
351 
352  uint32_t UcnVal = 0;
353  unsigned short UcnLen = 0;
354  FullSourceLoc Loc;
355 
356  if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
357  UcnLen, Loc, nullptr, Features, true)) {
358  HadError = true;
359  return 0;
360  }
361 
362  // UTF-16: 2 bytes for BMP, 4 bytes otherwise.
363  if (CharByteWidth == 2)
364  return UcnVal <= 0xFFFF ? 2 : 4;
365 
366  // UTF-8.
367  if (UcnVal < 0x80)
368  return 1;
369  if (UcnVal < 0x800)
370  return 2;
371  if (UcnVal < 0x10000)
372  return 3;
373  return 4;
374 }
375 
376 /// EncodeUCNEscape - Read the Universal Character Name, check constraints and
377 /// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
378 /// StringLiteralParser. When we decide to implement UCN's for identifiers,
379 /// we will likely rework our support for UCN's.
380 static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
381  const char *ThisTokEnd,
382  char *&ResultBuf, bool &HadError,
383  FullSourceLoc Loc, unsigned CharByteWidth,
384  DiagnosticsEngine *Diags,
385  const LangOptions &Features) {
386  typedef uint32_t UTF32;
387  UTF32 UcnVal = 0;
388  unsigned short UcnLen = 0;
389  if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen,
390  Loc, Diags, Features, true)) {
391  HadError = true;
392  return;
393  }
394 
395  assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) &&
396  "only character widths of 1, 2, or 4 bytes supported");
397 
398  (void)UcnLen;
399  assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
400 
401  if (CharByteWidth == 4) {
402  // FIXME: Make the type of the result buffer correct instead of
403  // using reinterpret_cast.
404  llvm::UTF32 *ResultPtr = reinterpret_cast<llvm::UTF32*>(ResultBuf);
405  *ResultPtr = UcnVal;
406  ResultBuf += 4;
407  return;
408  }
409 
410  if (CharByteWidth == 2) {
411  // FIXME: Make the type of the result buffer correct instead of
412  // using reinterpret_cast.
413  llvm::UTF16 *ResultPtr = reinterpret_cast<llvm::UTF16*>(ResultBuf);
414 
415  if (UcnVal <= (UTF32)0xFFFF) {
416  *ResultPtr = UcnVal;
417  ResultBuf += 2;
418  return;
419  }
420 
421  // Convert to UTF16.
422  UcnVal -= 0x10000;
423  *ResultPtr = 0xD800 + (UcnVal >> 10);
424  *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF);
425  ResultBuf += 4;
426  return;
427  }
428 
429  assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
430 
431  // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
432  // The conversion below was inspired by:
433  // http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
434  // First, we determine how many bytes the result will require.
435  typedef uint8_t UTF8;
436 
437  unsigned short bytesToWrite = 0;
438  if (UcnVal < (UTF32)0x80)
439  bytesToWrite = 1;
440  else if (UcnVal < (UTF32)0x800)
441  bytesToWrite = 2;
442  else if (UcnVal < (UTF32)0x10000)
443  bytesToWrite = 3;
444  else
445  bytesToWrite = 4;
446 
447  const unsigned byteMask = 0xBF;
448  const unsigned byteMark = 0x80;
449 
450  // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
451  // into the first byte, depending on how many bytes follow.
452  static const UTF8 firstByteMark[5] = {
453  0x00, 0x00, 0xC0, 0xE0, 0xF0
454  };
455  // Finally, we write the bytes into ResultBuf.
456  ResultBuf += bytesToWrite;
457  switch (bytesToWrite) { // note: everything falls through.
458  case 4:
459  *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
460  LLVM_FALLTHROUGH;
461  case 3:
462  *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
463  LLVM_FALLTHROUGH;
464  case 2:
465  *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
466  LLVM_FALLTHROUGH;
467  case 1:
468  *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
469  }
470  // Update the buffer.
471  ResultBuf += bytesToWrite;
472 }
473 
474 /// integer-constant: [C99 6.4.4.1]
475 /// decimal-constant integer-suffix
476 /// octal-constant integer-suffix
477 /// hexadecimal-constant integer-suffix
478 /// binary-literal integer-suffix [GNU, C++1y]
479 /// user-defined-integer-literal: [C++11 lex.ext]
480 /// decimal-literal ud-suffix
481 /// octal-literal ud-suffix
482 /// hexadecimal-literal ud-suffix
483 /// binary-literal ud-suffix [GNU, C++1y]
484 /// decimal-constant:
485 /// nonzero-digit
486 /// decimal-constant digit
487 /// octal-constant:
488 /// 0
489 /// octal-constant octal-digit
490 /// hexadecimal-constant:
491 /// hexadecimal-prefix hexadecimal-digit
492 /// hexadecimal-constant hexadecimal-digit
493 /// hexadecimal-prefix: one of
494 /// 0x 0X
495 /// binary-literal:
496 /// 0b binary-digit
497 /// 0B binary-digit
498 /// binary-literal binary-digit
499 /// integer-suffix:
500 /// unsigned-suffix [long-suffix]
501 /// unsigned-suffix [long-long-suffix]
502 /// long-suffix [unsigned-suffix]
503 /// long-long-suffix [unsigned-sufix]
504 /// nonzero-digit:
505 /// 1 2 3 4 5 6 7 8 9
506 /// octal-digit:
507 /// 0 1 2 3 4 5 6 7
508 /// hexadecimal-digit:
509 /// 0 1 2 3 4 5 6 7 8 9
510 /// a b c d e f
511 /// A B C D E F
512 /// binary-digit:
513 /// 0
514 /// 1
515 /// unsigned-suffix: one of
516 /// u U
517 /// long-suffix: one of
518 /// l L
519 /// long-long-suffix: one of
520 /// ll LL
521 ///
522 /// floating-constant: [C99 6.4.4.2]
523 /// TODO: add rules...
524 ///
526  SourceLocation TokLoc,
527  Preprocessor &PP)
528  : PP(PP), ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
529 
530  // This routine assumes that the range begin/end matches the regex for integer
531  // and FP constants (specifically, the 'pp-number' regex), and assumes that
532  // the byte at "*end" is both valid and not part of the regex. Because of
533  // this, it doesn't have to check for 'overscan' in various places.
534  assert(!isPreprocessingNumberBody(*ThisTokEnd) && "didn't maximally munch?");
535 
536  s = DigitsBegin = ThisTokBegin;
537  saw_exponent = false;
538  saw_period = false;
539  saw_ud_suffix = false;
540  saw_fixed_point_suffix = false;
541  isLong = false;
542  isUnsigned = false;
543  isLongLong = false;
544  isHalf = false;
545  isFloat = false;
546  isImaginary = false;
547  isFloat16 = false;
548  isFloat128 = false;
549  MicrosoftInteger = 0;
550  isFract = false;
551  isAccum = false;
552  hadError = false;
553 
554  if (*s == '0') { // parse radix
555  ParseNumberStartingWithZero(TokLoc);
556  if (hadError)
557  return;
558  } else { // the first digit is non-zero
559  radix = 10;
560  s = SkipDigits(s);
561  if (s == ThisTokEnd) {
562  // Done.
563  } else {
564  ParseDecimalOrOctalCommon(TokLoc);
565  if (hadError)
566  return;
567  }
568  }
569 
570  SuffixBegin = s;
571  checkSeparator(TokLoc, s, CSK_AfterDigits);
572 
573  // Initial scan to lookahead for fixed point suffix.
574  if (PP.getLangOpts().FixedPoint) {
575  for (const char *c = s; c != ThisTokEnd; ++c) {
576  if (*c == 'r' || *c == 'k' || *c == 'R' || *c == 'K') {
577  saw_fixed_point_suffix = true;
578  break;
579  }
580  }
581  }
582 
583  // Parse the suffix. At this point we can classify whether we have an FP or
584  // integer constant.
585  bool isFPConstant = isFloatingLiteral();
586 
587  // Loop over all of the characters of the suffix. If we see something bad,
588  // we break out of the loop.
589  for (; s != ThisTokEnd; ++s) {
590  switch (*s) {
591  case 'R':
592  case 'r':
593  if (!PP.getLangOpts().FixedPoint) break;
594  if (isFract || isAccum) break;
595  if (!(saw_period || saw_exponent)) break;
596  isFract = true;
597  continue;
598  case 'K':
599  case 'k':
600  if (!PP.getLangOpts().FixedPoint) break;
601  if (isFract || isAccum) break;
602  if (!(saw_period || saw_exponent)) break;
603  isAccum = true;
604  continue;
605  case 'h': // FP Suffix for "half".
606  case 'H':
607  // OpenCL Extension v1.2 s9.5 - h or H suffix for half type.
608  if (!(PP.getLangOpts().Half || PP.getLangOpts().FixedPoint)) break;
609  if (isIntegerLiteral()) break; // Error for integer constant.
610  if (isHalf || isFloat || isLong) break; // HH, FH, LH invalid.
611  isHalf = true;
612  continue; // Success.
613  case 'f': // FP Suffix for "float"
614  case 'F':
615  if (!isFPConstant) break; // Error for integer constant.
616  if (isHalf || isFloat || isLong || isFloat128)
617  break; // HF, FF, LF, QF invalid.
618 
619  // CUDA host and device may have different _Float16 support, therefore
620  // allows f16 literals to avoid false alarm.
621  // ToDo: more precise check for CUDA.
622  if ((PP.getTargetInfo().hasFloat16Type() || PP.getLangOpts().CUDA) &&
623  s + 2 < ThisTokEnd && s[1] == '1' && s[2] == '6') {
624  s += 2; // success, eat up 2 characters.
625  isFloat16 = true;
626  continue;
627  }
628 
629  isFloat = true;
630  continue; // Success.
631  case 'q': // FP Suffix for "__float128"
632  case 'Q':
633  if (!isFPConstant) break; // Error for integer constant.
634  if (isHalf || isFloat || isLong || isFloat128)
635  break; // HQ, FQ, LQ, QQ invalid.
636  isFloat128 = true;
637  continue; // Success.
638  case 'u':
639  case 'U':
640  if (isFPConstant) break; // Error for floating constant.
641  if (isUnsigned) break; // Cannot be repeated.
642  isUnsigned = true;
643  continue; // Success.
644  case 'l':
645  case 'L':
646  if (isLong || isLongLong) break; // Cannot be repeated.
647  if (isHalf || isFloat || isFloat128) break; // LH, LF, LQ invalid.
648 
649  // Check for long long. The L's need to be adjacent and the same case.
650  if (s[1] == s[0]) {
651  assert(s + 1 < ThisTokEnd && "didn't maximally munch?");
652  if (isFPConstant) break; // long long invalid for floats.
653  isLongLong = true;
654  ++s; // Eat both of them.
655  } else {
656  isLong = true;
657  }
658  continue; // Success.
659  case 'i':
660  case 'I':
661  if (PP.getLangOpts().MicrosoftExt) {
663  break;
664 
665  if (!isFPConstant) {
666  // Allow i8, i16, i32, and i64.
667  switch (s[1]) {
668  case '8':
669  s += 2; // i8 suffix
670  MicrosoftInteger = 8;
671  break;
672  case '1':
673  if (s[2] == '6') {
674  s += 3; // i16 suffix
675  MicrosoftInteger = 16;
676  }
677  break;
678  case '3':
679  if (s[2] == '2') {
680  s += 3; // i32 suffix
681  MicrosoftInteger = 32;
682  }
683  break;
684  case '6':
685  if (s[2] == '4') {
686  s += 3; // i64 suffix
687  MicrosoftInteger = 64;
688  }
689  break;
690  default:
691  break;
692  }
693  }
694  if (MicrosoftInteger) {
695  assert(s <= ThisTokEnd && "didn't maximally munch?");
696  break;
697  }
698  }
699  LLVM_FALLTHROUGH;
700  case 'j':
701  case 'J':
702  if (isImaginary) break; // Cannot be repeated.
703  isImaginary = true;
704  continue; // Success.
705  }
706  // If we reached here, there was an error or a ud-suffix.
707  break;
708  }
709 
710  // "i", "if", and "il" are user-defined suffixes in C++1y.
711  if (s != ThisTokEnd || isImaginary) {
712  // FIXME: Don't bother expanding UCNs if !tok.hasUCN().
713  expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
714  if (isValidUDSuffix(PP.getLangOpts(), UDSuffixBuf)) {
715  if (!isImaginary) {
716  // Any suffix pieces we might have parsed are actually part of the
717  // ud-suffix.
718  isLong = false;
719  isUnsigned = false;
720  isLongLong = false;
721  isFloat = false;
722  isFloat16 = false;
723  isHalf = false;
724  isImaginary = false;
725  MicrosoftInteger = 0;
726  saw_fixed_point_suffix = false;
727  isFract = false;
728  isAccum = false;
729  }
730 
731  saw_ud_suffix = true;
732  return;
733  }
734 
735  if (s != ThisTokEnd) {
736  // Report an error if there are any.
737  PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin),
738  diag::err_invalid_suffix_constant)
739  << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin) << isFPConstant;
740  hadError = true;
741  }
742  }
743 
744  if (!hadError && saw_fixed_point_suffix) {
745  assert(isFract || isAccum);
746  }
747 }
748 
749 /// ParseDecimalOrOctalCommon - This method is called for decimal or octal
750 /// numbers. It issues an error for illegal digits, and handles floating point
751 /// parsing. If it detects a floating point number, the radix is set to 10.
752 void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
753  assert((radix == 8 || radix == 10) && "Unexpected radix");
754 
755  // If we have a hex digit other than 'e' (which denotes a FP exponent) then
756  // the code is using an incorrect base.
757  if (isHexDigit(*s) && *s != 'e' && *s != 'E' &&
758  !isValidUDSuffix(PP.getLangOpts(), StringRef(s, ThisTokEnd - s))) {
759  PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
760  diag::err_invalid_digit) << StringRef(s, 1) << (radix == 8 ? 1 : 0);
761  hadError = true;
762  return;
763  }
764 
765  if (*s == '.') {
766  checkSeparator(TokLoc, s, CSK_AfterDigits);
767  s++;
768  radix = 10;
769  saw_period = true;
770  checkSeparator(TokLoc, s, CSK_BeforeDigits);
771  s = SkipDigits(s); // Skip suffix.
772  }
773  if (*s == 'e' || *s == 'E') { // exponent
774  checkSeparator(TokLoc, s, CSK_AfterDigits);
775  const char *Exponent = s;
776  s++;
777  radix = 10;
778  saw_exponent = true;
779  if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
780  const char *first_non_digit = SkipDigits(s);
781  if (containsDigits(s, first_non_digit)) {
782  checkSeparator(TokLoc, s, CSK_BeforeDigits);
783  s = first_non_digit;
784  } else {
785  if (!hadError) {
786  PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
787  diag::err_exponent_has_no_digits);
788  hadError = true;
789  }
790  return;
791  }
792  }
793 }
794 
795 /// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
796 /// suffixes as ud-suffixes, because the diagnostic experience is better if we
797 /// treat it as an invalid suffix.
799  StringRef Suffix) {
800  if (!LangOpts.CPlusPlus11 || Suffix.empty())
801  return false;
802 
803  // By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid.
804  if (Suffix[0] == '_')
805  return true;
806 
807  // In C++11, there are no library suffixes.
808  if (!LangOpts.CPlusPlus14)
809  return false;
810 
811  // In C++14, "s", "h", "min", "ms", "us", and "ns" are used in the library.
812  // Per tweaked N3660, "il", "i", and "if" are also used in the library.
813  // In C++2a "d" and "y" are used in the library.
814  return llvm::StringSwitch<bool>(Suffix)
815  .Cases("h", "min", "s", true)
816  .Cases("ms", "us", "ns", true)
817  .Cases("il", "i", "if", true)
818  .Cases("d", "y", LangOpts.CPlusPlus2a)
819  .Default(false);
820 }
821 
822 void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
823  const char *Pos,
824  CheckSeparatorKind IsAfterDigits) {
825  if (IsAfterDigits == CSK_AfterDigits) {
826  if (Pos == ThisTokBegin)
827  return;
828  --Pos;
829  } else if (Pos == ThisTokEnd)
830  return;
831 
832  if (isDigitSeparator(*Pos)) {
833  PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Pos - ThisTokBegin),
834  diag::err_digit_separator_not_between_digits)
835  << IsAfterDigits;
836  hadError = true;
837  }
838 }
839 
840 /// ParseNumberStartingWithZero - This method is called when the first character
841 /// of the number is found to be a zero. This means it is either an octal
842 /// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
843 /// a floating point number (01239.123e4). Eat the prefix, determining the
844 /// radix etc.
845 void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
846  assert(s[0] == '0' && "Invalid method call");
847  s++;
848 
849  int c1 = s[0];
850 
851  // Handle a hex number like 0x1234.
852  if ((c1 == 'x' || c1 == 'X') && (isHexDigit(s[1]) || s[1] == '.')) {
853  s++;
854  assert(s < ThisTokEnd && "didn't maximally munch?");
855  radix = 16;
856  DigitsBegin = s;
857  s = SkipHexDigits(s);
858  bool HasSignificandDigits = containsDigits(DigitsBegin, s);
859  if (s == ThisTokEnd) {
860  // Done.
861  } else if (*s == '.') {
862  s++;
863  saw_period = true;
864  const char *floatDigitsBegin = s;
865  s = SkipHexDigits(s);
866  if (containsDigits(floatDigitsBegin, s))
867  HasSignificandDigits = true;
868  if (HasSignificandDigits)
869  checkSeparator(TokLoc, floatDigitsBegin, CSK_BeforeDigits);
870  }
871 
872  if (!HasSignificandDigits) {
873  PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
874  diag::err_hex_constant_requires)
875  << PP.getLangOpts().CPlusPlus << 1;
876  hadError = true;
877  return;
878  }
879 
880  // A binary exponent can appear with or with a '.'. If dotted, the
881  // binary exponent is required.
882  if (*s == 'p' || *s == 'P') {
883  checkSeparator(TokLoc, s, CSK_AfterDigits);
884  const char *Exponent = s;
885  s++;
886  saw_exponent = true;
887  if (s != ThisTokEnd && (*s == '+' || *s == '-')) s++; // sign
888  const char *first_non_digit = SkipDigits(s);
889  if (!containsDigits(s, first_non_digit)) {
890  if (!hadError) {
891  PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
892  diag::err_exponent_has_no_digits);
893  hadError = true;
894  }
895  return;
896  }
897  checkSeparator(TokLoc, s, CSK_BeforeDigits);
898  s = first_non_digit;
899 
900  if (!PP.getLangOpts().HexFloats)
901  PP.Diag(TokLoc, PP.getLangOpts().CPlusPlus
902  ? diag::ext_hex_literal_invalid
903  : diag::ext_hex_constant_invalid);
904  else if (PP.getLangOpts().CPlusPlus17)
905  PP.Diag(TokLoc, diag::warn_cxx17_hex_literal);
906  } else if (saw_period) {
907  PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
908  diag::err_hex_constant_requires)
909  << PP.getLangOpts().CPlusPlus << 0;
910  hadError = true;
911  }
912  return;
913  }
914 
915  // Handle simple binary numbers 0b01010
916  if ((c1 == 'b' || c1 == 'B') && (s[1] == '0' || s[1] == '1')) {
917  // 0b101010 is a C++1y / GCC extension.
918  PP.Diag(TokLoc,
919  PP.getLangOpts().CPlusPlus14
920  ? diag::warn_cxx11_compat_binary_literal
921  : PP.getLangOpts().CPlusPlus
922  ? diag::ext_binary_literal_cxx14
923  : diag::ext_binary_literal);
924  ++s;
925  assert(s < ThisTokEnd && "didn't maximally munch?");
926  radix = 2;
927  DigitsBegin = s;
928  s = SkipBinaryDigits(s);
929  if (s == ThisTokEnd) {
930  // Done.
931  } else if (isHexDigit(*s) &&
933  StringRef(s, ThisTokEnd - s))) {
934  PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
935  diag::err_invalid_digit) << StringRef(s, 1) << 2;
936  hadError = true;
937  }
938  // Other suffixes will be diagnosed by the caller.
939  return;
940  }
941 
942  // For now, the radix is set to 8. If we discover that we have a
943  // floating point constant, the radix will change to 10. Octal floating
944  // point constants are not permitted (only decimal and hexadecimal).
945  radix = 8;
946  DigitsBegin = s;
947  s = SkipOctalDigits(s);
948  if (s == ThisTokEnd)
949  return; // Done, simple octal number like 01234
950 
951  // If we have some other non-octal digit that *is* a decimal digit, see if
952  // this is part of a floating point number like 094.123 or 09e1.
953  if (isDigit(*s)) {
954  const char *EndDecimal = SkipDigits(s);
955  if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
956  s = EndDecimal;
957  radix = 10;
958  }
959  }
960 
961  ParseDecimalOrOctalCommon(TokLoc);
962 }
963 
964 static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) {
965  switch (Radix) {
966  case 2:
967  return NumDigits <= 64;
968  case 8:
969  return NumDigits <= 64 / 3; // Digits are groups of 3 bits.
970  case 10:
971  return NumDigits <= 19; // floor(log10(2^64))
972  case 16:
973  return NumDigits <= 64 / 4; // Digits are groups of 4 bits.
974  default:
975  llvm_unreachable("impossible Radix");
976  }
977 }
978 
979 /// GetIntegerValue - Convert this numeric literal value to an APInt that
980 /// matches Val's input width. If there is an overflow, set Val to the low bits
981 /// of the result and return true. Otherwise, return false.
983  // Fast path: Compute a conservative bound on the maximum number of
984  // bits per digit in this radix. If we can't possibly overflow a
985  // uint64 based on that bound then do the simple conversion to
986  // integer. This avoids the expensive overflow checking below, and
987  // handles the common cases that matter (small decimal integers and
988  // hex/octal values which don't overflow).
989  const unsigned NumDigits = SuffixBegin - DigitsBegin;
990  if (alwaysFitsInto64Bits(radix, NumDigits)) {
991  uint64_t N = 0;
992  for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr)
993  if (!isDigitSeparator(*Ptr))
994  N = N * radix + llvm::hexDigitValue(*Ptr);
995 
996  // This will truncate the value to Val's input width. Simply check
997  // for overflow by comparing.
998  Val = N;
999  return Val.getZExtValue() != N;
1000  }
1001 
1002  Val = 0;
1003  const char *Ptr = DigitsBegin;
1004 
1005  llvm::APInt RadixVal(Val.getBitWidth(), radix);
1006  llvm::APInt CharVal(Val.getBitWidth(), 0);
1007  llvm::APInt OldVal = Val;
1008 
1009  bool OverflowOccurred = false;
1010  while (Ptr < SuffixBegin) {
1011  if (isDigitSeparator(*Ptr)) {
1012  ++Ptr;
1013  continue;
1014  }
1015 
1016  unsigned C = llvm::hexDigitValue(*Ptr++);
1017 
1018  // If this letter is out of bound for this radix, reject it.
1019  assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1020 
1021  CharVal = C;
1022 
1023  // Add the digit to the value in the appropriate radix. If adding in digits
1024  // made the value smaller, then this overflowed.
1025  OldVal = Val;
1026 
1027  // Multiply by radix, did overflow occur on the multiply?
1028  Val *= RadixVal;
1029  OverflowOccurred |= Val.udiv(RadixVal) != OldVal;
1030 
1031  // Add value, did overflow occur on the value?
1032  // (a + b) ult b <=> overflow
1033  Val += CharVal;
1034  OverflowOccurred |= Val.ult(CharVal);
1035  }
1036  return OverflowOccurred;
1037 }
1038 
1039 llvm::APFloat::opStatus
1041  using llvm::APFloat;
1042 
1043  unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin);
1044 
1045  llvm::SmallString<16> Buffer;
1046  StringRef Str(ThisTokBegin, n);
1047  if (Str.find('\'') != StringRef::npos) {
1048  Buffer.reserve(n);
1049  std::remove_copy_if(Str.begin(), Str.end(), std::back_inserter(Buffer),
1050  &isDigitSeparator);
1051  Str = Buffer;
1052  }
1053 
1054  return Result.convertFromString(Str, APFloat::rmNearestTiesToEven);
1055 }
1056 
1057 static inline bool IsExponentPart(char c) {
1058  return c == 'p' || c == 'P' || c == 'e' || c == 'E';
1059 }
1060 
1061 bool NumericLiteralParser::GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale) {
1062  assert(radix == 16 || radix == 10);
1063 
1064  // Find how many digits are needed to store the whole literal.
1065  unsigned NumDigits = SuffixBegin - DigitsBegin;
1066  if (saw_period) --NumDigits;
1067 
1068  // Initial scan of the exponent if it exists
1069  bool ExpOverflowOccurred = false;
1070  bool NegativeExponent = false;
1071  const char *ExponentBegin;
1072  uint64_t Exponent = 0;
1073  int64_t BaseShift = 0;
1074  if (saw_exponent) {
1075  const char *Ptr = DigitsBegin;
1076 
1077  while (!IsExponentPart(*Ptr)) ++Ptr;
1078  ExponentBegin = Ptr;
1079  ++Ptr;
1080  NegativeExponent = *Ptr == '-';
1081  if (NegativeExponent) ++Ptr;
1082 
1083  unsigned NumExpDigits = SuffixBegin - Ptr;
1084  if (alwaysFitsInto64Bits(radix, NumExpDigits)) {
1085  llvm::StringRef ExpStr(Ptr, NumExpDigits);
1086  llvm::APInt ExpInt(/*numBits=*/64, ExpStr, /*radix=*/10);
1087  Exponent = ExpInt.getZExtValue();
1088  } else {
1089  ExpOverflowOccurred = true;
1090  }
1091 
1092  if (NegativeExponent) BaseShift -= Exponent;
1093  else BaseShift += Exponent;
1094  }
1095 
1096  // Number of bits needed for decimal literal is
1097  // ceil(NumDigits * log2(10)) Integral part
1098  // + Scale Fractional part
1099  // + ceil(Exponent * log2(10)) Exponent
1100  // --------------------------------------------------
1101  // ceil((NumDigits + Exponent) * log2(10)) + Scale
1102  //
1103  // But for simplicity in handling integers, we can round up log2(10) to 4,
1104  // making:
1105  // 4 * (NumDigits + Exponent) + Scale
1106  //
1107  // Number of digits needed for hexadecimal literal is
1108  // 4 * NumDigits Integral part
1109  // + Scale Fractional part
1110  // + Exponent Exponent
1111  // --------------------------------------------------
1112  // (4 * NumDigits) + Scale + Exponent
1113  uint64_t NumBitsNeeded;
1114  if (radix == 10)
1115  NumBitsNeeded = 4 * (NumDigits + Exponent) + Scale;
1116  else
1117  NumBitsNeeded = 4 * NumDigits + Exponent + Scale;
1118 
1119  if (NumBitsNeeded > std::numeric_limits<unsigned>::max())
1120  ExpOverflowOccurred = true;
1121  llvm::APInt Val(static_cast<unsigned>(NumBitsNeeded), 0, /*isSigned=*/false);
1122 
1123  bool FoundDecimal = false;
1124 
1125  int64_t FractBaseShift = 0;
1126  const char *End = saw_exponent ? ExponentBegin : SuffixBegin;
1127  for (const char *Ptr = DigitsBegin; Ptr < End; ++Ptr) {
1128  if (*Ptr == '.') {
1129  FoundDecimal = true;
1130  continue;
1131  }
1132 
1133  // Normal reading of an integer
1134  unsigned C = llvm::hexDigitValue(*Ptr);
1135  assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1136 
1137  Val *= radix;
1138  Val += C;
1139 
1140  if (FoundDecimal)
1141  // Keep track of how much we will need to adjust this value by from the
1142  // number of digits past the radix point.
1143  --FractBaseShift;
1144  }
1145 
1146  // For a radix of 16, we will be multiplying by 2 instead of 16.
1147  if (radix == 16) FractBaseShift *= 4;
1148  BaseShift += FractBaseShift;
1149 
1150  Val <<= Scale;
1151 
1152  uint64_t Base = (radix == 16) ? 2 : 10;
1153  if (BaseShift > 0) {
1154  for (int64_t i = 0; i < BaseShift; ++i) {
1155  Val *= Base;
1156  }
1157  } else if (BaseShift < 0) {
1158  for (int64_t i = BaseShift; i < 0 && !Val.isNullValue(); ++i)
1159  Val = Val.udiv(Base);
1160  }
1161 
1162  bool IntOverflowOccurred = false;
1163  auto MaxVal = llvm::APInt::getMaxValue(StoreVal.getBitWidth());
1164  if (Val.getBitWidth() > StoreVal.getBitWidth()) {
1165  IntOverflowOccurred |= Val.ugt(MaxVal.zext(Val.getBitWidth()));
1166  StoreVal = Val.trunc(StoreVal.getBitWidth());
1167  } else if (Val.getBitWidth() < StoreVal.getBitWidth()) {
1168  IntOverflowOccurred |= Val.zext(MaxVal.getBitWidth()).ugt(MaxVal);
1169  StoreVal = Val.zext(StoreVal.getBitWidth());
1170  } else {
1171  StoreVal = Val;
1172  }
1173 
1174  return IntOverflowOccurred || ExpOverflowOccurred;
1175 }
1176 
1177 /// \verbatim
1178 /// user-defined-character-literal: [C++11 lex.ext]
1179 /// character-literal ud-suffix
1180 /// ud-suffix:
1181 /// identifier
1182 /// character-literal: [C++11 lex.ccon]
1183 /// ' c-char-sequence '
1184 /// u' c-char-sequence '
1185 /// U' c-char-sequence '
1186 /// L' c-char-sequence '
1187 /// u8' c-char-sequence ' [C++1z lex.ccon]
1188 /// c-char-sequence:
1189 /// c-char
1190 /// c-char-sequence c-char
1191 /// c-char:
1192 /// any member of the source character set except the single-quote ',
1193 /// backslash \, or new-line character
1194 /// escape-sequence
1195 /// universal-character-name
1196 /// escape-sequence:
1197 /// simple-escape-sequence
1198 /// octal-escape-sequence
1199 /// hexadecimal-escape-sequence
1200 /// simple-escape-sequence:
1201 /// one of \' \" \? \\ \a \b \f \n \r \t \v
1202 /// octal-escape-sequence:
1203 /// \ octal-digit
1204 /// \ octal-digit octal-digit
1205 /// \ octal-digit octal-digit octal-digit
1206 /// hexadecimal-escape-sequence:
1207 /// \x hexadecimal-digit
1208 /// hexadecimal-escape-sequence hexadecimal-digit
1209 /// universal-character-name: [C++11 lex.charset]
1210 /// \u hex-quad
1211 /// \U hex-quad hex-quad
1212 /// hex-quad:
1213 /// hex-digit hex-digit hex-digit hex-digit
1214 /// \endverbatim
1215 ///
1216 CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
1217  SourceLocation Loc, Preprocessor &PP,
1218  tok::TokenKind kind) {
1219  // At this point we know that the character matches the regex "(L|u|U)?'.*'".
1220  HadError = false;
1221 
1222  Kind = kind;
1223 
1224  const char *TokBegin = begin;
1225 
1226  // Skip over wide character determinant.
1227  if (Kind != tok::char_constant)
1228  ++begin;
1229  if (Kind == tok::utf8_char_constant)
1230  ++begin;
1231 
1232  // Skip over the entry quote.
1233  assert(begin[0] == '\'' && "Invalid token lexed");
1234  ++begin;
1235 
1236  // Remove an optional ud-suffix.
1237  if (end[-1] != '\'') {
1238  const char *UDSuffixEnd = end;
1239  do {
1240  --end;
1241  } while (end[-1] != '\'');
1242  // FIXME: Don't bother with this if !tok.hasUCN().
1243  expandUCNs(UDSuffixBuf, StringRef(end, UDSuffixEnd - end));
1244  UDSuffixOffset = end - TokBegin;
1245  }
1246 
1247  // Trim the ending quote.
1248  assert(end != begin && "Invalid token lexed");
1249  --end;
1250 
1251  // FIXME: The "Value" is an uint64_t so we can handle char literals of
1252  // up to 64-bits.
1253  // FIXME: This extensively assumes that 'char' is 8-bits.
1254  assert(PP.getTargetInfo().getCharWidth() == 8 &&
1255  "Assumes char is 8 bits");
1256  assert(PP.getTargetInfo().getIntWidth() <= 64 &&
1257  (PP.getTargetInfo().getIntWidth() & 7) == 0 &&
1258  "Assumes sizeof(int) on target is <= 64 and a multiple of char");
1259  assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
1260  "Assumes sizeof(wchar) on target is <= 64");
1261 
1262  SmallVector<uint32_t, 4> codepoint_buffer;
1263  codepoint_buffer.resize(end - begin);
1264  uint32_t *buffer_begin = &codepoint_buffer.front();
1265  uint32_t *buffer_end = buffer_begin + codepoint_buffer.size();
1266 
1267  // Unicode escapes representing characters that cannot be correctly
1268  // represented in a single code unit are disallowed in character literals
1269  // by this implementation.
1270  uint32_t largest_character_for_kind;
1271  if (tok::wide_char_constant == Kind) {
1272  largest_character_for_kind =
1273  0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth());
1274  } else if (tok::utf8_char_constant == Kind) {
1275  largest_character_for_kind = 0x7F;
1276  } else if (tok::utf16_char_constant == Kind) {
1277  largest_character_for_kind = 0xFFFF;
1278  } else if (tok::utf32_char_constant == Kind) {
1279  largest_character_for_kind = 0x10FFFF;
1280  } else {
1281  largest_character_for_kind = 0x7Fu;
1282  }
1283 
1284  while (begin != end) {
1285  // Is this a span of non-escape characters?
1286  if (begin[0] != '\\') {
1287  char const *start = begin;
1288  do {
1289  ++begin;
1290  } while (begin != end && *begin != '\\');
1291 
1292  char const *tmp_in_start = start;
1293  uint32_t *tmp_out_start = buffer_begin;
1294  llvm::ConversionResult res =
1295  llvm::ConvertUTF8toUTF32(reinterpret_cast<llvm::UTF8 const **>(&start),
1296  reinterpret_cast<llvm::UTF8 const *>(begin),
1297  &buffer_begin, buffer_end, llvm::strictConversion);
1298  if (res != llvm::conversionOK) {
1299  // If we see bad encoding for unprefixed character literals, warn and
1300  // simply copy the byte values, for compatibility with gcc and
1301  // older versions of clang.
1302  bool NoErrorOnBadEncoding = isAscii();
1303  unsigned Msg = diag::err_bad_character_encoding;
1304  if (NoErrorOnBadEncoding)
1305  Msg = diag::warn_bad_character_encoding;
1306  PP.Diag(Loc, Msg);
1307  if (NoErrorOnBadEncoding) {
1308  start = tmp_in_start;
1309  buffer_begin = tmp_out_start;
1310  for (; start != begin; ++start, ++buffer_begin)
1311  *buffer_begin = static_cast<uint8_t>(*start);
1312  } else {
1313  HadError = true;
1314  }
1315  } else {
1316  for (; tmp_out_start < buffer_begin; ++tmp_out_start) {
1317  if (*tmp_out_start > largest_character_for_kind) {
1318  HadError = true;
1319  PP.Diag(Loc, diag::err_character_too_large);
1320  }
1321  }
1322  }
1323 
1324  continue;
1325  }
1326  // Is this a Universal Character Name escape?
1327  if (begin[1] == 'u' || begin[1] == 'U') {
1328  unsigned short UcnLen = 0;
1329  if (!ProcessUCNEscape(TokBegin, begin, end, *buffer_begin, UcnLen,
1330  FullSourceLoc(Loc, PP.getSourceManager()),
1331  &PP.getDiagnostics(), PP.getLangOpts(), true)) {
1332  HadError = true;
1333  } else if (*buffer_begin > largest_character_for_kind) {
1334  HadError = true;
1335  PP.Diag(Loc, diag::err_character_too_large);
1336  }
1337 
1338  ++buffer_begin;
1339  continue;
1340  }
1341  unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo());
1342  uint64_t result =
1343  ProcessCharEscape(TokBegin, begin, end, HadError,
1344  FullSourceLoc(Loc,PP.getSourceManager()),
1345  CharWidth, &PP.getDiagnostics(), PP.getLangOpts());
1346  *buffer_begin++ = result;
1347  }
1348 
1349  unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front();
1350 
1351  if (NumCharsSoFar > 1) {
1352  if (isWide())
1353  PP.Diag(Loc, diag::warn_extraneous_char_constant);
1354  else if (isAscii() && NumCharsSoFar == 4)
1355  PP.Diag(Loc, diag::ext_four_char_character_literal);
1356  else if (isAscii())
1357  PP.Diag(Loc, diag::ext_multichar_character_literal);
1358  else
1359  PP.Diag(Loc, diag::err_multichar_utf_character_literal);
1360  IsMultiChar = true;
1361  } else {
1362  IsMultiChar = false;
1363  }
1364 
1365  llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
1366 
1367  // Narrow character literals act as though their value is concatenated
1368  // in this implementation, but warn on overflow.
1369  bool multi_char_too_long = false;
1370  if (isAscii() && isMultiChar()) {
1371  LitVal = 0;
1372  for (size_t i = 0; i < NumCharsSoFar; ++i) {
1373  // check for enough leading zeros to shift into
1374  multi_char_too_long |= (LitVal.countLeadingZeros() < 8);
1375  LitVal <<= 8;
1376  LitVal = LitVal + (codepoint_buffer[i] & 0xFF);
1377  }
1378  } else if (NumCharsSoFar > 0) {
1379  // otherwise just take the last character
1380  LitVal = buffer_begin[-1];
1381  }
1382 
1383  if (!HadError && multi_char_too_long) {
1384  PP.Diag(Loc, diag::warn_char_constant_too_large);
1385  }
1386 
1387  // Transfer the value from APInt to uint64_t
1388  Value = LitVal.getZExtValue();
1389 
1390  // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
1391  // if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple
1392  // character constants are not sign extended in the this implementation:
1393  // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
1394  if (isAscii() && NumCharsSoFar == 1 && (Value & 128) &&
1395  PP.getLangOpts().CharIsSigned)
1396  Value = (signed char)Value;
1397 }
1398 
1399 /// \verbatim
1400 /// string-literal: [C++0x lex.string]
1401 /// encoding-prefix " [s-char-sequence] "
1402 /// encoding-prefix R raw-string
1403 /// encoding-prefix:
1404 /// u8
1405 /// u
1406 /// U
1407 /// L
1408 /// s-char-sequence:
1409 /// s-char
1410 /// s-char-sequence s-char
1411 /// s-char:
1412 /// any member of the source character set except the double-quote ",
1413 /// backslash \, or new-line character
1414 /// escape-sequence
1415 /// universal-character-name
1416 /// raw-string:
1417 /// " d-char-sequence ( r-char-sequence ) d-char-sequence "
1418 /// r-char-sequence:
1419 /// r-char
1420 /// r-char-sequence r-char
1421 /// r-char:
1422 /// any member of the source character set, except a right parenthesis )
1423 /// followed by the initial d-char-sequence (which may be empty)
1424 /// followed by a double quote ".
1425 /// d-char-sequence:
1426 /// d-char
1427 /// d-char-sequence d-char
1428 /// d-char:
1429 /// any member of the basic source character set except:
1430 /// space, the left parenthesis (, the right parenthesis ),
1431 /// the backslash \, and the control characters representing horizontal
1432 /// tab, vertical tab, form feed, and newline.
1433 /// escape-sequence: [C++0x lex.ccon]
1434 /// simple-escape-sequence
1435 /// octal-escape-sequence
1436 /// hexadecimal-escape-sequence
1437 /// simple-escape-sequence:
1438 /// one of \' \" \? \\ \a \b \f \n \r \t \v
1439 /// octal-escape-sequence:
1440 /// \ octal-digit
1441 /// \ octal-digit octal-digit
1442 /// \ octal-digit octal-digit octal-digit
1443 /// hexadecimal-escape-sequence:
1444 /// \x hexadecimal-digit
1445 /// hexadecimal-escape-sequence hexadecimal-digit
1446 /// universal-character-name:
1447 /// \u hex-quad
1448 /// \U hex-quad hex-quad
1449 /// hex-quad:
1450 /// hex-digit hex-digit hex-digit hex-digit
1451 /// \endverbatim
1452 ///
1455  Preprocessor &PP, bool Complain)
1456  : SM(PP.getSourceManager()), Features(PP.getLangOpts()),
1457  Target(PP.getTargetInfo()), Diags(Complain ? &PP.getDiagnostics() :nullptr),
1458  MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown),
1459  ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) {
1460  init(StringToks);
1461 }
1462 
1463 void StringLiteralParser::init(ArrayRef<Token> StringToks){
1464  // The literal token may have come from an invalid source location (e.g. due
1465  // to a PCH error), in which case the token length will be 0.
1466  if (StringToks.empty() || StringToks[0].getLength() < 2)
1467  return DiagnoseLexingError(SourceLocation());
1468 
1469  // Scan all of the string portions, remember the max individual token length,
1470  // computing a bound on the concatenated string length, and see whether any
1471  // piece is a wide-string. If any of the string portions is a wide-string
1472  // literal, the result is a wide-string literal [C99 6.4.5p4].
1473  assert(!StringToks.empty() && "expected at least one token");
1474  MaxTokenLength = StringToks[0].getLength();
1475  assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
1476  SizeBound = StringToks[0].getLength()-2; // -2 for "".
1477  Kind = StringToks[0].getKind();
1478 
1479  hadError = false;
1480 
1481  // Implement Translation Phase #6: concatenation of string literals
1482  /// (C99 5.1.1.2p1). The common case is only one string fragment.
1483  for (unsigned i = 1; i != StringToks.size(); ++i) {
1484  if (StringToks[i].getLength() < 2)
1485  return DiagnoseLexingError(StringToks[i].getLocation());
1486 
1487  // The string could be shorter than this if it needs cleaning, but this is a
1488  // reasonable bound, which is all we need.
1489  assert(StringToks[i].getLength() >= 2 && "literal token is invalid!");
1490  SizeBound += StringToks[i].getLength()-2; // -2 for "".
1491 
1492  // Remember maximum string piece length.
1493  if (StringToks[i].getLength() > MaxTokenLength)
1494  MaxTokenLength = StringToks[i].getLength();
1495 
1496  // Remember if we see any wide or utf-8/16/32 strings.
1497  // Also check for illegal concatenations.
1498  if (StringToks[i].isNot(Kind) && StringToks[i].isNot(tok::string_literal)) {
1499  if (isAscii()) {
1500  Kind = StringToks[i].getKind();
1501  } else {
1502  if (Diags)
1503  Diags->Report(StringToks[i].getLocation(),
1504  diag::err_unsupported_string_concat);
1505  hadError = true;
1506  }
1507  }
1508  }
1509 
1510  // Include space for the null terminator.
1511  ++SizeBound;
1512 
1513  // TODO: K&R warning: "traditional C rejects string constant concatenation"
1514 
1515  // Get the width in bytes of char/wchar_t/char16_t/char32_t
1516  CharByteWidth = getCharWidth(Kind, Target);
1517  assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
1518  CharByteWidth /= 8;
1519 
1520  // The output buffer size needs to be large enough to hold wide characters.
1521  // This is a worst-case assumption which basically corresponds to L"" "long".
1522  SizeBound *= CharByteWidth;
1523 
1524  // Size the temporary buffer to hold the result string data.
1525  ResultBuf.resize(SizeBound);
1526 
1527  // Likewise, but for each string piece.
1528  SmallString<512> TokenBuf;
1529  TokenBuf.resize(MaxTokenLength);
1530 
1531  // Loop over all the strings, getting their spelling, and expanding them to
1532  // wide strings as appropriate.
1533  ResultPtr = &ResultBuf[0]; // Next byte to fill in.
1534 
1535  Pascal = false;
1536 
1537  SourceLocation UDSuffixTokLoc;
1538 
1539  for (unsigned i = 0, e = StringToks.size(); i != e; ++i) {
1540  const char *ThisTokBuf = &TokenBuf[0];
1541  // Get the spelling of the token, which eliminates trigraphs, etc. We know
1542  // that ThisTokBuf points to a buffer that is big enough for the whole token
1543  // and 'spelled' tokens can only shrink.
1544  bool StringInvalid = false;
1545  unsigned ThisTokLen =
1546  Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features,
1547  &StringInvalid);
1548  if (StringInvalid)
1549  return DiagnoseLexingError(StringToks[i].getLocation());
1550 
1551  const char *ThisTokBegin = ThisTokBuf;
1552  const char *ThisTokEnd = ThisTokBuf+ThisTokLen;
1553 
1554  // Remove an optional ud-suffix.
1555  if (ThisTokEnd[-1] != '"') {
1556  const char *UDSuffixEnd = ThisTokEnd;
1557  do {
1558  --ThisTokEnd;
1559  } while (ThisTokEnd[-1] != '"');
1560 
1561  StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd);
1562 
1563  if (UDSuffixBuf.empty()) {
1564  if (StringToks[i].hasUCN())
1565  expandUCNs(UDSuffixBuf, UDSuffix);
1566  else
1567  UDSuffixBuf.assign(UDSuffix);
1568  UDSuffixToken = i;
1569  UDSuffixOffset = ThisTokEnd - ThisTokBuf;
1570  UDSuffixTokLoc = StringToks[i].getLocation();
1571  } else {
1572  SmallString<32> ExpandedUDSuffix;
1573  if (StringToks[i].hasUCN()) {
1574  expandUCNs(ExpandedUDSuffix, UDSuffix);
1575  UDSuffix = ExpandedUDSuffix;
1576  }
1577 
1578  // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the
1579  // result of a concatenation involving at least one user-defined-string-
1580  // literal, all the participating user-defined-string-literals shall
1581  // have the same ud-suffix.
1582  if (UDSuffixBuf != UDSuffix) {
1583  if (Diags) {
1584  SourceLocation TokLoc = StringToks[i].getLocation();
1585  Diags->Report(TokLoc, diag::err_string_concat_mixed_suffix)
1586  << UDSuffixBuf << UDSuffix
1587  << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc)
1588  << SourceRange(TokLoc, TokLoc);
1589  }
1590  hadError = true;
1591  }
1592  }
1593  }
1594 
1595  // Strip the end quote.
1596  --ThisTokEnd;
1597 
1598  // TODO: Input character set mapping support.
1599 
1600  // Skip marker for wide or unicode strings.
1601  if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') {
1602  ++ThisTokBuf;
1603  // Skip 8 of u8 marker for utf8 strings.
1604  if (ThisTokBuf[0] == '8')
1605  ++ThisTokBuf;
1606  }
1607 
1608  // Check for raw string
1609  if (ThisTokBuf[0] == 'R') {
1610  ThisTokBuf += 2; // skip R"
1611 
1612  const char *Prefix = ThisTokBuf;
1613  while (ThisTokBuf[0] != '(')
1614  ++ThisTokBuf;
1615  ++ThisTokBuf; // skip '('
1616 
1617  // Remove same number of characters from the end
1618  ThisTokEnd -= ThisTokBuf - Prefix;
1619  assert(ThisTokEnd >= ThisTokBuf && "malformed raw string literal");
1620 
1621  // C++14 [lex.string]p4: A source-file new-line in a raw string literal
1622  // results in a new-line in the resulting execution string-literal.
1623  StringRef RemainingTokenSpan(ThisTokBuf, ThisTokEnd - ThisTokBuf);
1624  while (!RemainingTokenSpan.empty()) {
1625  // Split the string literal on \r\n boundaries.
1626  size_t CRLFPos = RemainingTokenSpan.find("\r\n");
1627  StringRef BeforeCRLF = RemainingTokenSpan.substr(0, CRLFPos);
1628  StringRef AfterCRLF = RemainingTokenSpan.substr(CRLFPos);
1629 
1630  // Copy everything before the \r\n sequence into the string literal.
1631  if (CopyStringFragment(StringToks[i], ThisTokBegin, BeforeCRLF))
1632  hadError = true;
1633 
1634  // Point into the \n inside the \r\n sequence and operate on the
1635  // remaining portion of the literal.
1636  RemainingTokenSpan = AfterCRLF.substr(1);
1637  }
1638  } else {
1639  if (ThisTokBuf[0] != '"') {
1640  // The file may have come from PCH and then changed after loading the
1641  // PCH; Fail gracefully.
1642  return DiagnoseLexingError(StringToks[i].getLocation());
1643  }
1644  ++ThisTokBuf; // skip "
1645 
1646  // Check if this is a pascal string
1647  if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
1648  ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') {
1649 
1650  // If the \p sequence is found in the first token, we have a pascal string
1651  // Otherwise, if we already have a pascal string, ignore the first \p
1652  if (i == 0) {
1653  ++ThisTokBuf;
1654  Pascal = true;
1655  } else if (Pascal)
1656  ThisTokBuf += 2;
1657  }
1658 
1659  while (ThisTokBuf != ThisTokEnd) {
1660  // Is this a span of non-escape characters?
1661  if (ThisTokBuf[0] != '\\') {
1662  const char *InStart = ThisTokBuf;
1663  do {
1664  ++ThisTokBuf;
1665  } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
1666 
1667  // Copy the character span over.
1668  if (CopyStringFragment(StringToks[i], ThisTokBegin,
1669  StringRef(InStart, ThisTokBuf - InStart)))
1670  hadError = true;
1671  continue;
1672  }
1673  // Is this a Universal Character Name escape?
1674  if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') {
1675  EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
1676  ResultPtr, hadError,
1677  FullSourceLoc(StringToks[i].getLocation(), SM),
1678  CharByteWidth, Diags, Features);
1679  continue;
1680  }
1681  // Otherwise, this is a non-UCN escape character. Process it.
1682  unsigned ResultChar =
1683  ProcessCharEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, hadError,
1684  FullSourceLoc(StringToks[i].getLocation(), SM),
1685  CharByteWidth*8, Diags, Features);
1686 
1687  if (CharByteWidth == 4) {
1688  // FIXME: Make the type of the result buffer correct instead of
1689  // using reinterpret_cast.
1690  llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultPtr);
1691  *ResultWidePtr = ResultChar;
1692  ResultPtr += 4;
1693  } else if (CharByteWidth == 2) {
1694  // FIXME: Make the type of the result buffer correct instead of
1695  // using reinterpret_cast.
1696  llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultPtr);
1697  *ResultWidePtr = ResultChar & 0xFFFF;
1698  ResultPtr += 2;
1699  } else {
1700  assert(CharByteWidth == 1 && "Unexpected char width");
1701  *ResultPtr++ = ResultChar & 0xFF;
1702  }
1703  }
1704  }
1705  }
1706 
1707  if (Pascal) {
1708  if (CharByteWidth == 4) {
1709  // FIXME: Make the type of the result buffer correct instead of
1710  // using reinterpret_cast.
1711  llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultBuf.data());
1712  ResultWidePtr[0] = GetNumStringChars() - 1;
1713  } else if (CharByteWidth == 2) {
1714  // FIXME: Make the type of the result buffer correct instead of
1715  // using reinterpret_cast.
1716  llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultBuf.data());
1717  ResultWidePtr[0] = GetNumStringChars() - 1;
1718  } else {
1719  assert(CharByteWidth == 1 && "Unexpected char width");
1720  ResultBuf[0] = GetNumStringChars() - 1;
1721  }
1722 
1723  // Verify that pascal strings aren't too large.
1724  if (GetStringLength() > 256) {
1725  if (Diags)
1726  Diags->Report(StringToks.front().getLocation(),
1727  diag::err_pascal_string_too_long)
1728  << SourceRange(StringToks.front().getLocation(),
1729  StringToks.back().getLocation());
1730  hadError = true;
1731  return;
1732  }
1733  } else if (Diags) {
1734  // Complain if this string literal has too many characters.
1735  unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
1736 
1737  if (GetNumStringChars() > MaxChars)
1738  Diags->Report(StringToks.front().getLocation(),
1739  diag::ext_string_too_long)
1740  << GetNumStringChars() << MaxChars
1741  << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
1742  << SourceRange(StringToks.front().getLocation(),
1743  StringToks.back().getLocation());
1744  }
1745 }
1746 
1747 static const char *resyncUTF8(const char *Err, const char *End) {
1748  if (Err == End)
1749  return End;
1750  End = Err + std::min<unsigned>(llvm::getNumBytesForUTF8(*Err), End-Err);
1751  while (++Err != End && (*Err & 0xC0) == 0x80)
1752  ;
1753  return Err;
1754 }
1755 
1756 /// This function copies from Fragment, which is a sequence of bytes
1757 /// within Tok's contents (which begin at TokBegin) into ResultPtr.
1758 /// Performs widening for multi-byte characters.
1759 bool StringLiteralParser::CopyStringFragment(const Token &Tok,
1760  const char *TokBegin,
1761  StringRef Fragment) {
1762  const llvm::UTF8 *ErrorPtrTmp;
1763  if (ConvertUTF8toWide(CharByteWidth, Fragment, ResultPtr, ErrorPtrTmp))
1764  return false;
1765 
1766  // If we see bad encoding for unprefixed string literals, warn and
1767  // simply copy the byte values, for compatibility with gcc and older
1768  // versions of clang.
1769  bool NoErrorOnBadEncoding = isAscii();
1770  if (NoErrorOnBadEncoding) {
1771  memcpy(ResultPtr, Fragment.data(), Fragment.size());
1772  ResultPtr += Fragment.size();
1773  }
1774 
1775  if (Diags) {
1776  const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
1777 
1778  FullSourceLoc SourceLoc(Tok.getLocation(), SM);
1779  const DiagnosticBuilder &Builder =
1780  Diag(Diags, Features, SourceLoc, TokBegin,
1781  ErrorPtr, resyncUTF8(ErrorPtr, Fragment.end()),
1782  NoErrorOnBadEncoding ? diag::warn_bad_string_encoding
1783  : diag::err_bad_string_encoding);
1784 
1785  const char *NextStart = resyncUTF8(ErrorPtr, Fragment.end());
1786  StringRef NextFragment(NextStart, Fragment.end()-NextStart);
1787 
1788  // Decode into a dummy buffer.
1789  SmallString<512> Dummy;
1790  Dummy.reserve(Fragment.size() * CharByteWidth);
1791  char *Ptr = Dummy.data();
1792 
1793  while (!ConvertUTF8toWide(CharByteWidth, NextFragment, Ptr, ErrorPtrTmp)) {
1794  const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
1795  NextStart = resyncUTF8(ErrorPtr, Fragment.end());
1796  Builder << MakeCharSourceRange(Features, SourceLoc, TokBegin,
1797  ErrorPtr, NextStart);
1798  NextFragment = StringRef(NextStart, Fragment.end()-NextStart);
1799  }
1800  }
1801  return !NoErrorOnBadEncoding;
1802 }
1803 
1804 void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) {
1805  hadError = true;
1806  if (Diags)
1807  Diags->Report(Loc, diag::err_lexing_string);
1808 }
1809 
1810 /// getOffsetOfStringByte - This function returns the offset of the
1811 /// specified byte of the string data represented by Token. This handles
1812 /// advancing over escape sequences in the string.
1814  unsigned ByteNo) const {
1815  // Get the spelling of the token.
1816  SmallString<32> SpellingBuffer;
1817  SpellingBuffer.resize(Tok.getLength());
1818 
1819  bool StringInvalid = false;
1820  const char *SpellingPtr = &SpellingBuffer[0];
1821  unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features,
1822  &StringInvalid);
1823  if (StringInvalid)
1824  return 0;
1825 
1826  const char *SpellingStart = SpellingPtr;
1827  const char *SpellingEnd = SpellingPtr+TokLen;
1828 
1829  // Handle UTF-8 strings just like narrow strings.
1830  if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8')
1831  SpellingPtr += 2;
1832 
1833  assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
1834  SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
1835 
1836  // For raw string literals, this is easy.
1837  if (SpellingPtr[0] == 'R') {
1838  assert(SpellingPtr[1] == '"' && "Should be a raw string literal!");
1839  // Skip 'R"'.
1840  SpellingPtr += 2;
1841  while (*SpellingPtr != '(') {
1842  ++SpellingPtr;
1843  assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal");
1844  }
1845  // Skip '('.
1846  ++SpellingPtr;
1847  return SpellingPtr - SpellingStart + ByteNo;
1848  }
1849 
1850  // Skip over the leading quote
1851  assert(SpellingPtr[0] == '"' && "Should be a string literal!");
1852  ++SpellingPtr;
1853 
1854  // Skip over bytes until we find the offset we're looking for.
1855  while (ByteNo) {
1856  assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
1857 
1858  // Step over non-escapes simply.
1859  if (*SpellingPtr != '\\') {
1860  ++SpellingPtr;
1861  --ByteNo;
1862  continue;
1863  }
1864 
1865  // Otherwise, this is an escape character. Advance over it.
1866  bool HadError = false;
1867  if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U') {
1868  const char *EscapePtr = SpellingPtr;
1869  unsigned Len = MeasureUCNEscape(SpellingStart, SpellingPtr, SpellingEnd,
1870  1, Features, HadError);
1871  if (Len > ByteNo) {
1872  // ByteNo is somewhere within the escape sequence.
1873  SpellingPtr = EscapePtr;
1874  break;
1875  }
1876  ByteNo -= Len;
1877  } else {
1878  ProcessCharEscape(SpellingStart, SpellingPtr, SpellingEnd, HadError,
1879  FullSourceLoc(Tok.getLocation(), SM),
1880  CharByteWidth*8, Diags, Features);
1881  --ByteNo;
1882  }
1883  assert(!HadError && "This method isn't valid on erroneous strings");
1884  }
1885 
1886  return SpellingPtr-SpellingStart;
1887 }
1888 
1889 /// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
1890 /// suffixes as ud-suffixes, because the diagnostic experience is better if we
1891 /// treat it as an invalid suffix.
1893  StringRef Suffix) {
1894  return NumericLiteralParser::isValidUDSuffix(LangOpts, Suffix) ||
1895  Suffix == "sv";
1896 }
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer...
Definition: Lexer.cpp:388
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, unsigned DiagID)
Produce a diagnostic highlighting some portion of a literal.
unsigned getOffsetOfStringByte(const Token &TheTok, unsigned ByteNo) const
getOffsetOfStringByte - This function returns the offset of the specified byte of the string data rep...
StringLiteralParser(ArrayRef< Token > StringToks, Preprocessor &PP, bool Complain=true)
unsigned GetNumStringChars() const
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1297
unsigned getCharWidth() const
Definition: TargetInfo.h:382
unsigned GetStringLength() const
static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits)
unsigned getChar32Width() const
getChar32Width/Align - Return the size of &#39;char32_t&#39; for this target, in bits.
Definition: TargetInfo.h:570
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:904
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
__DEVICE__ int max(int __a, int __b)
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:49
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:903
static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, const char *ThisTokEnd, unsigned CharByteWidth, const LangOptions &Features, bool &HadError)
MeasureUCNEscape - Determine the number of bytes within the resulting string which this UCN will occu...
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
const FormatToken & Tok
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:149
static bool isValidUDSuffix(const LangOptions &LangOpts, StringRef Suffix)
Determine whether a suffix is a valid ud-suffix.
NumericLiteralParser(StringRef TokSpelling, SourceLocation TokLoc, Preprocessor &PP)
integer-constant: [C99 6.4.4.1] decimal-constant integer-suffix octal-constant integer-suffix hexadec...
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Characters, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token...
Definition: Lexer.h:348
static bool isValidUDSuffix(const LangOptions &LangOpts, StringRef Suffix)
Determine whether a suffix is a valid ud-suffix.
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1045
Exposes information about the current target.
Definition: TargetInfo.h:161
bool GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale)
GetFixedPointValue - Convert this numeric literal value into a scaled integer that represents this va...
CharLiteralParser(const char *begin, const char *end, SourceLocation Loc, Preprocessor &PP, tok::TokenKind kind)
Defines the clang::LangOptions interface.
SourceLocation End
Represents a character-granular source range.
unsigned getIntWidth() const
getIntWidth/Align - Return the size of &#39;signed int&#39; and &#39;unsigned int&#39; for this target, in bits.
Definition: TargetInfo.h:395
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
Defines the clang::Preprocessor interface.
const SourceManager & getManager() const
SourceLocation Begin
static const char * resyncUTF8(const char *Err, const char *End)
static void appendCodePoint(unsigned Codepoint, llvm::SmallVectorImpl< char > &Str)
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1586
static CharSourceRange getCharRange(SourceRange R)
SourceManager & getSourceManager() const
Definition: Preprocessor.h:907
unsigned getWCharWidth() const
getWCharWidth/Align - Return the size of &#39;wchar_t&#39; for this target, in bits.
Definition: TargetInfo.h:560
bool GetIntegerValue(llvm::APInt &Val)
GetIntegerValue - Convert this numeric literal value to an APInt that matches Val&#39;s input width...
#define false
Definition: stdbool.h:17
Kind
Encodes a location in the source.
llvm::APFloat::opStatus GetFloatValue(llvm::APFloat &Result)
GetFloatValue - Convert this numeric literal to a floating value, using the specified APFloat fltSema...
unsigned getChar16Width() const
getChar16Width/Align - Return the size of &#39;char16_t&#39; for this target, in bits.
Definition: TargetInfo.h:565
static unsigned ProcessCharEscape(const char *ThisTokBegin, const char *&ThisTokBuf, const char *ThisTokEnd, bool &HadError, FullSourceLoc Loc, unsigned CharWidth, DiagnosticsEngine *Diags, const LangOptions &Features)
ProcessCharEscape - Parse a standard C escape sequence, which can occur in either a character or a st...
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:24
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
LLVM_READONLY bool isPrintable(unsigned char c)
Return true if this character is an ASCII printable character; that is, a character that should take ...
Definition: CharInfo.h:139
static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, const char *ThisTokEnd, char *&ResultBuf, bool &HadError, FullSourceLoc Loc, unsigned CharByteWidth, DiagnosticsEngine *Diags, const LangOptions &Features)
EncodeUCNEscape - Read the Universal Character Name, check constraints and convert the UTF32 to UTF8 ...
Dataflow Directional Tag Classes.
virtual bool hasFloat16Type() const
Determine whether the _Float16 type is supported on this target.
Definition: TargetInfo.h:533
unsigned getLength() const
Definition: Token.h:129
LLVM_READONLY bool isHexDigit(unsigned char c)
Return true if this character is an ASCII hex digit: [0-9a-fA-F].
Definition: CharInfo.h:123
LLVM_READONLY bool isDigit(unsigned char c)
Return true if this character is an ASCII digit: [0-9].
Definition: CharInfo.h:93
Defines the clang::SourceLocation class and associated facilities.
static bool IsExponentPart(char c)
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:900
static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target)
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Definition: DiagnosticIDs.h:60
__DEVICE__ int min(int __a, int __b)
Defines the clang::TargetInfo interface.
A SourceLocation and its associated SourceManager.
A trivial tuple used to represent a source range.
static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, const char *ThisTokEnd, uint32_t &UcnVal, unsigned short &UcnLen, FullSourceLoc Loc, DiagnosticsEngine *Diags, const LangOptions &Features, bool in_char_string_literal=false)
ProcessUCNEscape - Read the Universal Character Name, check constraints and return the UTF32...
LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
Definition: CharInfo.h:147
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
static CharSourceRange MakeCharSourceRange(const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:124