2002-04-24 Matt Hiller <hiller@redhat.com>

	* cpplex.c: Remove conditional #undef of MULTIBYTE_CHARS.
	* c-lex.c: Ditto.

	* cpplex.c (skip_line_comment): Process comment one multibyte
	character at a time rather than one char at a time, if
	appropriate.
	(parse_string): Process string one multibyte character at a time
	rather than one char at a time, if appropriate.
	* c-lex.c (lex_string): Lex and copy multibyte strings
	appropriately.
	* cpplib.h (cppchar_t): Change to unsigned.

git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@52737 138bc75d-0d04-0410-961f-82ee72b054a4

diff --git a/gcc/c-lex.c b/gcc/c-lex.c
index 8a76dd3..f2f8c8c 100644 (file)
--- a/gcc/c-lex.c
+++ b/gcc/c-lex.c
@@ -40,13 +40,6 @@ Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include "splay-tree.h"
 #include "debug.h"
 
-/* MULTIBYTE_CHARS support only works for native compilers.
-   ??? Ideally what we want is to model widechar support after
-   the current floating point support.  */
-#ifdef CROSS_COMPILE
-#undef MULTIBYTE_CHARS
-#endif
-
 #ifdef MULTIBYTE_CHARS
 #include "mbchar.h"
 #include <locale.h>
@@ -1282,8 +1275,8 @@ lex_string (str, len, wide)
          c = cpp_parse_escape (parse_in, &p, limit, mask);
        }
        
-      /* Add this single character into the buffer either as a wchar_t
-        or as a single byte.  */
+      /* Add this single character into the buffer either as a wchar_t,
+        a multibyte sequence, or as a single byte.  */
       if (wide)
        {
          unsigned charwidth = TYPE_PRECISION (char_type_node);
@@ -1304,6 +1297,16 @@ lex_string (str, len, wide)
            }
          q += WCHAR_BYTES;
        }
+#ifdef MULTIBYTE_CHARS
+      else if (char_len > 1)
+       {
+         /* We're dealing with a multibyte character. */
+         for ( ; char_len >0; --char_len)
+           {
+             *q++ = *(p - char_len);
+           }
+       }
+#endif
       else
        {
          *q++ = c;

diff --git a/gcc/cpplex.c b/gcc/cpplex.c
index eed9ffd..6a44b86 100644 (file)
--- a/gcc/cpplex.c
+++ b/gcc/cpplex.c
@@ -25,13 +25,6 @@ Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
 #include "cpplib.h"
 #include "cpphash.h"
 
-/* MULTIBYTE_CHARS support only works for native compilers.
-   ??? Ideally what we want is to model widechar support after
-   the current floating point support.  */
-#ifdef CROSS_COMPILE
-#undef MULTIBYTE_CHARS
-#endif
-
 #ifdef MULTIBYTE_CHARS
 #include "mbchar.h"
 #include <locale.h>
@@ -312,14 +305,39 @@ skip_line_comment (pfile)
   cpp_buffer *buffer = pfile->buffer;
   unsigned int orig_line = pfile->line;
   cppchar_t c;
+#ifdef MULTIBYTE_CHARS
+  wchar_t wc;
+  int char_len;
+#endif
 
   pfile->state.lexing_comment = 1;
+#ifdef MULTIBYTE_CHARS
+  /* Reset multibyte conversion state.  */
+  (void) local_mbtowc (NULL, NULL, 0);
+#endif
   do
     {
       if (buffer->cur == buffer->rlimit)
        goto at_eof;
 
+#ifdef MULTIBYTE_CHARS
+      char_len = local_mbtowc (&wc, (const char *) buffer->cur,
+                              buffer->rlimit - buffer->cur);
+      if (char_len == -1)
+       {
+         cpp_error (pfile, DL_WARNING,
+                    "ignoring invalid multibyte character");
+         char_len = 1;
+         c = *buffer->cur++;
+       }
+      else
+       {
+         buffer->cur += char_len;
+         c = wc;
+       }
+#else
       c = *buffer->cur++;
+#endif
       if (c == '?' || c == '\\')
        c = skip_escaped_newlines (pfile);
     }
@@ -617,10 +635,18 @@ parse_string (pfile, token, terminator)
   unsigned char *dest, *limit;
   cppchar_t c;
   bool warned_nulls = false;
+#ifdef MULTIBYTE_CHARS
+  wchar_t wc;
+  int char_len;
+#endif
 
   dest = BUFF_FRONT (pfile->u_buff);
   limit = BUFF_LIMIT (pfile->u_buff);
 
+#ifdef MULTIBYTE_CHARS
+  /* Reset multibyte conversion state.  */
+  (void) local_mbtowc (NULL, NULL, 0);
+#endif
   for (;;)
     {
       /* We need room for another char, possibly the terminating NUL.  */
@@ -632,8 +658,26 @@ parse_string (pfile, token, terminator)
          limit = BUFF_LIMIT (pfile->u_buff);
        }
 
-      /* Handle trigraphs, escaped newlines etc.  */
+#ifdef MULTIBYTE_CHARS
+      char_len = local_mbtowc (&wc, (const char *) buffer->cur,
+                              buffer->rlimit - buffer->cur);
+      if (char_len == -1)
+       {
+         cpp_error (pfile, DL_WARNING,
+                      "ignoring invalid multibyte character");
+         char_len = 1;
+         c = *buffer->cur++;
+       }
+      else
+       {
+         buffer->cur += char_len;
+         c = wc;
+       }
+#else
       c = *buffer->cur++;
+#endif
+
+      /* Handle trigraphs, escaped newlines etc.  */
       if (c == '?' || c == '\\')
        c = skip_escaped_newlines (pfile);
 
@@ -666,8 +710,15 @@ parse_string (pfile, token, terminator)
                         "null character(s) preserved in literal");
            }
        }
-
-      *dest++ = c;
+#ifdef MULTIBYTE_CHARS
+      if (char_len > 1)
+       {
+         for ( ; char_len > 0; --char_len)
+           *dest++ = (*buffer->cur - char_len);
+       }
+      else
+#endif
+       *dest++ = c;
     }
 
   *dest = '\0';

diff --git a/gcc/cpplib.h b/gcc/cpplib.h
index 359326e..765d65a 100644 (file)
--- a/gcc/cpplib.h
+++ b/gcc/cpplib.h
@@ -187,9 +187,9 @@ struct cpp_token
   } val;
 };
 
-/* A standalone character.  We may want to make it unsigned for the
-   same reason we use unsigned char - to avoid signedness issues.  */
-typedef int cppchar_t;
+/* A standalone character.  It is unsigned for the same reason we use
+   unsigned char - to avoid signedness issues.  */
+typedef unsigned int cppchar_t;
 
 /* Values for opts.dump_macros.
   dump_only means inhibit output of the preprocessed text