Implement C++1y digit separator proposal (' as a digit separator). This is not

yet approved by full committee, but was unanimously supported by EWG.

llvm-svn: 191417

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -159,6 +159,11 @@ def err_invalid_suffix_integer_constant : Error<
   "invalid suffix '%0' on integer constant">;
 def err_invalid_suffix_float_constant : Error<
   "invalid suffix '%0' on floating constant">;
+def warn_cxx11_compat_digit_separator : Warning<
+  "digit separators are incompatible with C++ standards before C++1y">,
+  InGroup<CXXPre1yCompat>, DefaultIgnore;
+def err_digit_separator_not_between_digits : Error<
+  "digit separator cannot appear at %select{start|end}0 of digit sequence">;
 def warn_extraneous_char_constant : Warning<
   "extraneous characters in character constant ignored">;
 def warn_char_constant_too_large : Warning<

clang/include/clang/Lex/LiteralSupport.h

@@ -100,10 +100,16 @@ private:
 
   void ParseNumberStartingWithZero(SourceLocation TokLoc);
 
+  static bool isDigitSeparator(char C) { return C == '\''; }
+
+  /// \brief Ensure that we don't have a digit separator here.
+  void checkSeparator(SourceLocation TokLoc, const char *Pos,
+                      bool IsAfterDigits);
+
   /// SkipHexDigits - Read and skip over any hex digits, up to End.
   /// Return a pointer to the first non-hex digit or End.
   const char *SkipHexDigits(const char *ptr) {
-    while (ptr != ThisTokEnd && isHexDigit(*ptr))
+    while (ptr != ThisTokEnd && (isHexDigit(*ptr) || isDigitSeparator(*ptr)))
       ptr++;
     return ptr;
   }
@@ -111,7 +117,8 @@ private:
   /// SkipOctalDigits - Read and skip over any octal digits, up to End.
   /// Return a pointer to the first non-hex digit or End.
   const char *SkipOctalDigits(const char *ptr) {
-    while (ptr != ThisTokEnd && ((*ptr >= '0') && (*ptr <= '7')))
+    while (ptr != ThisTokEnd &&
+           ((*ptr >= '0' && *ptr <= '7') || isDigitSeparator(*ptr)))
       ptr++;
     return ptr;
   }
@@ -119,7 +126,7 @@ private:
   /// SkipDigits - Read and skip over any digits, up to End.
   /// Return a pointer to the first non-hex digit or End.
   const char *SkipDigits(const char *ptr) {
-    while (ptr != ThisTokEnd && isDigit(*ptr))
+    while (ptr != ThisTokEnd && (isDigit(*ptr) || isDigitSeparator(*ptr)))
       ptr++;
     return ptr;
   }
@@ -127,7 +134,8 @@ private:
   /// SkipBinaryDigits - Read and skip over any binary digits, up to End.
   /// Return a pointer to the first non-binary digit or End.
   const char *SkipBinaryDigits(const char *ptr) {
-    while (ptr != ThisTokEnd && (*ptr == '0' || *ptr == '1'))
+    while (ptr != ThisTokEnd &&
+           (*ptr == '0' || *ptr == '1' || isDigitSeparator(*ptr)))
       ptr++;
     return ptr;
   }

clang/lib/Lex/Lexer.cpp

@@ -1606,6 +1606,18 @@ bool Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
       return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
   }
 
+  // If we have a digit separator, continue.
+  if (C == '\'' && getLangOpts().CPlusPlus1y) {
+    unsigned NextSize;
+    char Next = getCharAndSizeNoWarn(CurPtr + Size, NextSize, getLangOpts());
+    if (isAlphanumeric(Next)) {
+      if (!isLexingRawMode())
+        Diag(CurPtr, diag::warn_cxx11_compat_digit_separator);
+      CurPtr = ConsumeChar(CurPtr, Size, Result);
+      return LexNumericConstant(Result, CurPtr);
+    }
+  }
+
   // Update the location of token as well as BufferPtr.
   const char *TokStart = BufferPtr;
   FormTokenWithChars(Result, CurPtr, tok::numeric_constant);

clang/lib/Lex/LiteralSupport.cpp

@@ -498,15 +498,19 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
       hadError = true;
       return;
     } else if (*s == '.') {
+      checkSeparator(TokLoc, s, true);
       s++;
       saw_period = true;
+      checkSeparator(TokLoc, s, false);
       s = SkipDigits(s);
     }
     if ((*s == 'e' || *s == 'E')) { // exponent
+      checkSeparator(TokLoc, s, true);
       const char *Exponent = s;
       s++;
       saw_exponent = true;
       if (*s == '+' || *s == '-')  s++; // sign
+      checkSeparator(TokLoc, s, false);
       const char *first_non_digit = SkipDigits(s);
       if (first_non_digit != s) {
         s = first_non_digit;
@@ -520,6 +524,7 @@ NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
   }
 
   SuffixBegin = s;
+  checkSeparator(TokLoc, s, true);
 
   // Parse the suffix.  At this point we can classify whether we have an FP or
   // integer constant.
@@ -676,6 +681,21 @@ bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
       .Default(false);
 }
 
+void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
+                                          const char *Pos, bool IsAfterDigits) {
+  if (IsAfterDigits) {
+    assert(Pos != ThisTokBegin);
+    --Pos;
+  } else {
+    assert(Pos != ThisTokEnd);
+  }
+
+  if (isDigitSeparator(*Pos))
+    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Pos - ThisTokBegin),
+            diag::err_digit_separator_not_between_digits)
+      << IsAfterDigits;
+}
+
 /// ParseNumberStartingWithZero - This method is called when the first character
 /// of the number is found to be a zero.  This means it is either an octal
 /// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
@@ -736,7 +756,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
   }
 
   // Handle simple binary numbers 0b01010
-  if (*s == 'b' || *s == 'B') {
+  if ((*s == 'b' || *s == 'B') && (s[1] == '0' || s[1] == '1')) {
     // 0b101010 is a C++1y / GCC extension.
     PP.Diag(TokLoc,
             PP.getLangOpts().CPlusPlus1y
@@ -840,7 +860,8 @@ bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
   if (alwaysFitsInto64Bits(radix, NumDigits)) {
     uint64_t N = 0;
     for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr)
-      N = N * radix + llvm::hexDigitValue(*Ptr);
+      if (!isDigitSeparator(*Ptr))
+        N = N * radix + llvm::hexDigitValue(*Ptr);
 
     // This will truncate the value to Val's input width. Simply check
     // for overflow by comparing.
@@ -857,6 +878,11 @@ bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
 
   bool OverflowOccurred = false;
   while (Ptr < SuffixBegin) {
+    if (isDigitSeparator(*Ptr)) {
+      ++Ptr;
+      continue;
+    }
+
     unsigned C = llvm::hexDigitValue(*Ptr++);
 
     // If this letter is out of bound for this radix, reject it.
@@ -885,8 +911,17 @@ NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
   using llvm::APFloat;
 
   unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin);
-  return Result.convertFromString(StringRef(ThisTokBegin, n),
-                                  APFloat::rmNearestTiesToEven);
+
+  llvm::SmallString<16> Buffer;
+  StringRef Str(ThisTokBegin, n);
+  if (Str.find('\'') != StringRef::npos) {
+    Buffer.reserve(n);
+    std::remove_copy_if(Str.begin(), Str.end(), std::back_inserter(Buffer),
+                        &isDigitSeparator);
+    Str = Buffer;
+  }
+
+  return Result.convertFromString(Str, APFloat::rmNearestTiesToEven);
 }
 
 

clang/test/Lexer/cxx1y_binary_literal.cpp

@@ -17,3 +17,4 @@ int k1 = 0b1234; // expected-error {{invalid digit '2' in binary constant}}
 // we'll need to rework our binary literal parsing rules.
 int k2 = 0b10010f; // expected-error {{invalid digit 'f' in binary constant}}
 int k3 = 0b10010g; // expected-error {{invalid suffix 'g' on integer constant}}
+int k4 = 0b; // expected-error {{invalid digit 'b' in octal constant}}

clang/test/Lexer/cxx1y_digit_separators.cpp

@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -std=c++1y -verify %s
+
+int operator""ms(unsigned long long); // expected-warning {{reserved}}
+float operator""ms(long double); // expected-warning {{reserved}}
+
+namespace integral {
+  static_assert(1'2'3 == 12'3, "");
+  static_assert(1'000'000 == 0xf'4240, "");
+  static_assert(0'004'000'000 == 0x10'0000, "");
+  static_assert(0b0101'0100 == 0x54, "");
+
+  int a = 123'; //'; // expected-error {{expected ';'}}
+  int b = 0'xff; // expected-error {{digit separator cannot appear at end of digit sequence}} expected-error {{suffix 'xff' on integer}}
+  int c = 0x'ff; // expected-error {{suffix 'x'ff' on integer}}
+  int d = 0'1234; // ok, octal
+  int e = 0'b1010; // expected-error {{digit 'b' in octal constant}}
+  int f = 0b'1010; // expected-error {{invalid digit 'b' in octal}}
+  int g = 123'ms; // expected-error {{digit separator cannot appear at end of digit sequence}}
+
+  // FIXME: not yet known if _ after ' will be permitted.
+  int z = 0'123'_foo; //'; // expected-error {{expected ';'}}
+}
+
+namespace floating {
+  static_assert(0'123.456'7 == 123.4567, "");
+  static_assert(1e1'0 == 10'000'000'000, "");
+
+  float a = 1'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
+  float b = 1'0e1;
+  float c = 1.'0e1; // expected-error {{digit separator cannot appear at start of digit sequence}}
+  float d = 1.0'e1; // expected-error {{digit separator cannot appear at end of digit sequence}}
+  float e = 1e'1; // expected-error {{digit separator cannot appear at start of digit sequence}}
+  float f = 1e1'ms; // expected-error {{digit separator cannot appear at end of digit sequence}}
+}

clang/test/SemaCXX/cxx98-compat.cpp

@@ -1,8 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++11 -Wc++98-compat -verify %s
-// RUN: %clang_cc1 -fsyntax-only -std=c++11 %s
-// RUN: %clang_cc1 -fsyntax-only -std=c++1y -Wc++98-c++11-compat -verify %s -DCXX1YCOMPAT
-
-#ifndef CXX1YCOMPAT
+// RUN: %clang_cc1 -fsyntax-only -std=c++1y -Wc++98-compat -verify %s -DCXX1YCOMPAT
 
 namespace std {
   struct type_info;
@@ -381,7 +378,6 @@ namespace rdar11736429 {
     X x; // expected-warning{{union member 'x' with a non-trivial constructor is incompatible with C++98}}
   };
 }
-#endif
 
 template<typename T> T var = T(10);
 #ifdef CXX1YCOMPAT
@@ -455,3 +451,6 @@ template<> int B::v<int> = 10;
 template int B::v<int>;
 float fsvar = B::v<float>;
 
+#ifdef CXX1YCOMPAT
+int digit_seps = 123'456; // expected-warning {{digit separators are incompatible with C++ standards before C++1y}}
+#endif