1 /* CPP Library - charsets
2 Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2006, 2008, 2009
3 Free Software Foundation, Inc.
5 Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
7 This program is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by the
9 Free Software Foundation; either version 3, or (at your option) any
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
26 /* Character set handling for C-family languages.
28 Terminological note: In what follows, "charset" or "character set"
29 will be taken to mean both an abstract set of characters and an
30 encoding for that set.
32 The C99 standard discusses two character sets: source and execution.
33 The source character set is used for internal processing in translation
34 phases 1 through 4; the execution character set is used thereafter.
35 Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
36 character encodings (see 3.7.2, 3.7.3 for the standardese meanings
37 of these terms). Furthermore, the "basic character set" (listed in
38 5.2.1p3) is to be encoded in each with values one byte wide, and is
39 to appear in the initial shift state.
41 It is not explicitly mentioned, but there is also a "wide execution
42 character set" used to encode wide character constants and wide
43 string literals; this is supposed to be the result of applying the
44 standard library function mbstowcs() to an equivalent narrow string
45 (6.4.5p5). However, the behavior of hexadecimal and octal
46 \-escapes is at odds with this; they are supposed to be translated
47 directly to wchar_t values (6.4.4.4p5,6).
49 The source character set is not necessarily the character set used
50 to encode physical source files on disk; translation phase 1 converts
51 from whatever that encoding is to the source character set.
53 The presence of universal character names in C99 (6.4.3 et seq.)
54 forces the source character set to be isomorphic to ISO 10646,
55 that is, Unicode. There is no such constraint on the execution
56 character set; note also that the conversion from source to
57 execution character set does not occur for identifiers (5.1.1.2p1#5).
59 For convenience of implementation, the source character set's
60 encoding of the basic character set should be identical to the
61 execution character set OF THE HOST SYSTEM's encoding of the basic
62 character set, and it should not be a state-dependent encoding.
64 cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
65 depending on whether the host is based on ASCII or EBCDIC (see
66 respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
67 Technical Report #16). With limited exceptions, it relies on the
68 system library's iconv() primitive to do charset conversion
69 (specified in SUSv2). */
72 /* Make certain that the uses of iconv(), iconv_open(), iconv_close()
73 below, which are guarded only by if statements with compile-time
74 constant conditions, do not cause link errors. */
75 #define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
76 #define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
77 #define iconv_close(x) (void)0
81 #if HOST_CHARSET == HOST_CHARSET_ASCII
82 #define SOURCE_CHARSET "UTF-8"
83 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e
84 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
85 #define SOURCE_CHARSET "UTF-EBCDIC"
86 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF
88 #error "Unrecognized basic host character set"
95 /* This structure is used for a resizable string buffer throughout. */
96 /* Don't call it strbuf, as that conflicts with unistd.h on systems
97 such as DYNIX/ptx where unistd.h includes stropts.h. */
105 /* This is enough to hold any string that fits on a single 80-column
106 line, even if iconv quadruples its size (e.g. conversion from
107 ASCII to UTF-32) rounded up to a power of two. */
108 #define OUTBUF_BLOCK_SIZE 256
110 /* Conversions between UTF-8 and UTF-16/32 are implemented by custom
111 logic. This is because a depressing number of systems lack iconv,
112 or have have iconv libraries that do not do these conversions, so
113 we need a fallback implementation for them. To ensure the fallback
114 doesn't break due to neglect, it is used on all systems.
116 UTF-32 encoding is nice and simple: a four-byte binary number,
117 constrained to the range 00000000-7FFFFFFF to avoid questions of
118 signedness. We do have to cope with big- and little-endian
121 UTF-16 encoding uses two-byte binary numbers, again in big- and
122 little-endian variants, for all values in the 00000000-0000FFFF
123 range. Values in the 00010000-0010FFFF range are encoded as pairs
124 of two-byte numbers, called "surrogate pairs": given a number S in
125 this range, it is mapped to a pair (H, L) as follows:
127 H = (S - 0x10000) / 0x400 + 0xD800
128 L = (S - 0x10000) % 0x400 + 0xDC00
130 Two-byte values in the D800...DFFF range are ill-formed except as a
131 component of a surrogate pair. Even if the encoding within a
132 two-byte value is little-endian, the H member of the surrogate pair
135 There is no way to encode values in the 00110000-7FFFFFFF range,
136 which is not currently a problem as there are no assigned code
137 points in that range; however, the author expects that it will
138 eventually become necessary to abandon UTF-16 due to this
139 limitation. Note also that, because of these pairs, UTF-16 does
140 not meet the requirements of the C standard for a wide character
141 encoding (see 3.7.3 and 6.4.4.4p11).
143 UTF-8 encoding looks like this:
145 value range encoded as
146 00000000-0000007F 0xxxxxxx
147 00000080-000007FF 110xxxxx 10xxxxxx
148 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
149 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
150 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
151 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
153 Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
154 which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
155 never occur. Note also that any value that can be encoded by a
156 given row of the table can also be encoded by all successive rows,
157 but this is not done; only the shortest possible encoding for any
158 given value is valid. For instance, the character 07C0 could be
159 encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
160 FC 80 80 80 9F 80. Only the first is valid.
162 An implementation note: the transformation from UTF-16 to UTF-8, or
163 vice versa, is easiest done by using UTF-32 as an intermediary. */
165 /* Internal primitives which go from an UTF-8 byte stream to native-endian
166 UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
167 operation in several places below. */
169 one_utf8_to_cppchar (const uchar **inbufp, size_t *inbytesleftp,
172 static const uchar masks[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
173 static const uchar patns[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
176 const uchar *inbuf = *inbufp;
179 if (*inbytesleftp < 1)
191 /* The number of leading 1-bits in the first byte indicates how many
193 for (nbytes = 2; nbytes < 7; nbytes++)
194 if ((c & ~masks[nbytes-1]) == patns[nbytes-1])
199 if (*inbytesleftp < nbytes)
202 c = (c & masks[nbytes-1]);
204 for (i = 1; i < nbytes; i++)
206 cppchar_t n = *inbuf++;
207 if ((n & 0xC0) != 0x80)
209 c = ((c << 6) + (n & 0x3F));
212 /* Make sure the shortest possible encoding was used. */
213 if (c <= 0x7F && nbytes > 1) return EILSEQ;
214 if (c <= 0x7FF && nbytes > 2) return EILSEQ;
215 if (c <= 0xFFFF && nbytes > 3) return EILSEQ;
216 if (c <= 0x1FFFFF && nbytes > 4) return EILSEQ;
217 if (c <= 0x3FFFFFF && nbytes > 5) return EILSEQ;
219 /* Make sure the character is valid. */
220 if (c > 0x7FFFFFFF || (c >= 0xD800 && c <= 0xDFFF)) return EILSEQ;
224 *inbytesleftp -= nbytes;
229 one_cppchar_to_utf8 (cppchar_t c, uchar **outbufp, size_t *outbytesleftp)
231 static const uchar masks[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
232 static const uchar limits[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
234 uchar buf[6], *p = &buf[6];
235 uchar *outbuf = *outbufp;
244 *--p = ((c & 0x3F) | 0x80);
248 while (c >= 0x3F || (c & limits[nbytes-1]));
249 *--p = (c | masks[nbytes-1]);
252 if (*outbytesleftp < nbytes)
257 *outbytesleftp -= nbytes;
262 /* The following four functions transform one character between the two
263 encodings named in the function name. All have the signature
264 int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
265 uchar **outbufp, size_t *outbytesleftp)
267 BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
268 interpreted as a boolean indicating whether big-endian or
269 little-endian encoding is to be used for the member of the pair
272 INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
275 The return value is either 0 for success, or an errno value for
276 failure, which may be E2BIG (need more space), EILSEQ (ill-formed
277 input sequence), ir EINVAL (incomplete input sequence). */
280 one_utf8_to_utf32 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
281 uchar **outbufp, size_t *outbytesleftp)
287 /* Check for space first, since we know exactly how much we need. */
288 if (*outbytesleftp < 4)
291 rval = one_utf8_to_cppchar (inbufp, inbytesleftp, &s);
296 outbuf[bigend ? 3 : 0] = (s & 0x000000FF);
297 outbuf[bigend ? 2 : 1] = (s & 0x0000FF00) >> 8;
298 outbuf[bigend ? 1 : 2] = (s & 0x00FF0000) >> 16;
299 outbuf[bigend ? 0 : 3] = (s & 0xFF000000) >> 24;
307 one_utf32_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
308 uchar **outbufp, size_t *outbytesleftp)
314 if (*inbytesleftp < 4)
319 s = inbuf[bigend ? 0 : 3] << 24;
320 s += inbuf[bigend ? 1 : 2] << 16;
321 s += inbuf[bigend ? 2 : 1] << 8;
322 s += inbuf[bigend ? 3 : 0];
324 if (s >= 0x7FFFFFFF || (s >= 0xD800 && s <= 0xDFFF))
327 rval = one_cppchar_to_utf8 (s, outbufp, outbytesleftp);
337 one_utf8_to_utf16 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
338 uchar **outbufp, size_t *outbytesleftp)
342 const uchar *save_inbuf = *inbufp;
343 size_t save_inbytesleft = *inbytesleftp;
344 uchar *outbuf = *outbufp;
346 rval = one_utf8_to_cppchar (inbufp, inbytesleftp, &s);
352 *inbufp = save_inbuf;
353 *inbytesleftp = save_inbytesleft;
359 if (*outbytesleftp < 2)
361 *inbufp = save_inbuf;
362 *inbytesleftp = save_inbytesleft;
365 outbuf[bigend ? 1 : 0] = (s & 0x00FF);
366 outbuf[bigend ? 0 : 1] = (s & 0xFF00) >> 8;
376 if (*outbytesleftp < 4)
378 *inbufp = save_inbuf;
379 *inbytesleftp = save_inbytesleft;
383 hi = (s - 0x10000) / 0x400 + 0xD800;
384 lo = (s - 0x10000) % 0x400 + 0xDC00;
386 /* Even if we are little-endian, put the high surrogate first.
387 ??? Matches practice? */
388 outbuf[bigend ? 1 : 0] = (hi & 0x00FF);
389 outbuf[bigend ? 0 : 1] = (hi & 0xFF00) >> 8;
390 outbuf[bigend ? 3 : 2] = (lo & 0x00FF);
391 outbuf[bigend ? 2 : 3] = (lo & 0xFF00) >> 8;
400 one_utf16_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
401 uchar **outbufp, size_t *outbytesleftp)
404 const uchar *inbuf = *inbufp;
407 if (*inbytesleftp < 2)
409 s = inbuf[bigend ? 0 : 1] << 8;
410 s += inbuf[bigend ? 1 : 0];
412 /* Low surrogate without immediately preceding high surrogate is invalid. */
413 if (s >= 0xDC00 && s <= 0xDFFF)
415 /* High surrogate must have a following low surrogate. */
416 else if (s >= 0xD800 && s <= 0xDBFF)
418 cppchar_t hi = s, lo;
419 if (*inbytesleftp < 4)
422 lo = inbuf[bigend ? 2 : 3] << 8;
423 lo += inbuf[bigend ? 3 : 2];
425 if (lo < 0xDC00 || lo > 0xDFFF)
428 s = (hi - 0xD800) * 0x400 + (lo - 0xDC00) + 0x10000;
431 rval = one_cppchar_to_utf8 (s, outbufp, outbytesleftp);
435 /* Success - update the input pointers (one_cppchar_to_utf8 has done
436 the output pointers for us). */
450 /* Helper routine for the next few functions. The 'const' on
451 one_conversion means that we promise not to modify what function is
452 pointed to, which lets the inliner see through it. */
455 conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *,
457 iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to)
461 size_t inbytesleft, outbytesleft;
466 outbuf = to->text + to->len;
467 outbytesleft = to->asize - to->len;
472 rval = one_conversion (cd, &inbuf, &inbytesleft,
473 &outbuf, &outbytesleft);
474 while (inbytesleft && !rval);
476 if (__builtin_expect (inbytesleft == 0, 1))
478 to->len = to->asize - outbytesleft;
487 outbytesleft += OUTBUF_BLOCK_SIZE;
488 to->asize += OUTBUF_BLOCK_SIZE;
489 to->text = XRESIZEVEC (uchar, to->text, to->asize);
490 outbuf = to->text + to->asize - outbytesleft;
495 /* These functions convert entire strings between character sets.
496 They all have the signature
498 bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
500 The input string FROM is converted as specified by the function
501 name plus the iconv descriptor CD (which may be fake), and the
502 result appended to TO. On any error, false is returned, otherwise true. */
504 /* These four use the custom conversion code above. */
506 convert_utf8_utf16 (iconv_t cd, const uchar *from, size_t flen,
507 struct _cpp_strbuf *to)
509 return conversion_loop (one_utf8_to_utf16, cd, from, flen, to);
513 convert_utf8_utf32 (iconv_t cd, const uchar *from, size_t flen,
514 struct _cpp_strbuf *to)
516 return conversion_loop (one_utf8_to_utf32, cd, from, flen, to);
520 convert_utf16_utf8 (iconv_t cd, const uchar *from, size_t flen,
521 struct _cpp_strbuf *to)
523 return conversion_loop (one_utf16_to_utf8, cd, from, flen, to);
527 convert_utf32_utf8 (iconv_t cd, const uchar *from, size_t flen,
528 struct _cpp_strbuf *to)
530 return conversion_loop (one_utf32_to_utf8, cd, from, flen, to);
533 /* Identity conversion, used when we have no alternative. */
535 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED,
536 const uchar *from, size_t flen, struct _cpp_strbuf *to)
538 if (to->len + flen > to->asize)
540 to->asize = to->len + flen;
541 to->text = XRESIZEVEC (uchar, to->text, to->asize);
543 memcpy (to->text + to->len, from, flen);
548 /* And this one uses the system iconv primitive. It's a little
549 different, since iconv's interface is a little different. */
552 #define CONVERT_ICONV_GROW_BUFFER \
554 outbytesleft += OUTBUF_BLOCK_SIZE; \
555 to->asize += OUTBUF_BLOCK_SIZE; \
556 to->text = XRESIZEVEC (uchar, to->text, to->asize); \
557 outbuf = (char *)to->text + to->asize - outbytesleft; \
561 convert_using_iconv (iconv_t cd, const uchar *from, size_t flen,
562 struct _cpp_strbuf *to)
564 ICONV_CONST char *inbuf;
566 size_t inbytesleft, outbytesleft;
568 /* Reset conversion descriptor and check that it is valid. */
569 if (iconv (cd, 0, 0, 0, 0) == (size_t)-1)
572 inbuf = (ICONV_CONST char *)from;
574 outbuf = (char *)to->text + to->len;
575 outbytesleft = to->asize - to->len;
579 iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
580 if (__builtin_expect (inbytesleft == 0, 1))
582 /* Close out any shift states, returning to the initial state. */
583 if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
588 CONVERT_ICONV_GROW_BUFFER;
589 if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
593 to->len = to->asize - outbytesleft;
599 CONVERT_ICONV_GROW_BUFFER;
603 #define convert_using_iconv 0 /* prevent undefined symbol error below */
606 /* Arrange for the above custom conversion logic to be used automatically
607 when conversion between a suitable pair of character sets is requested. */
609 #define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
610 CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
618 static const struct conversion conversion_tab[] = {
619 { "UTF-8/UTF-32LE", convert_utf8_utf32, (iconv_t)0 },
620 { "UTF-8/UTF-32BE", convert_utf8_utf32, (iconv_t)1 },
621 { "UTF-8/UTF-16LE", convert_utf8_utf16, (iconv_t)0 },
622 { "UTF-8/UTF-16BE", convert_utf8_utf16, (iconv_t)1 },
623 { "UTF-32LE/UTF-8", convert_utf32_utf8, (iconv_t)0 },
624 { "UTF-32BE/UTF-8", convert_utf32_utf8, (iconv_t)1 },
625 { "UTF-16LE/UTF-8", convert_utf16_utf8, (iconv_t)0 },
626 { "UTF-16BE/UTF-8", convert_utf16_utf8, (iconv_t)1 },
629 /* Subroutine of cpp_init_iconv: initialize and return a
630 cset_converter structure for conversion from FROM to TO. If
631 iconv_open() fails, issue an error and return an identity
632 converter. Silently return an identity converter if FROM and TO
634 static struct cset_converter
635 init_iconv_desc (cpp_reader *pfile, const char *to, const char *from)
637 struct cset_converter ret;
641 if (!strcasecmp (to, from))
643 ret.func = convert_no_conversion;
644 ret.cd = (iconv_t) -1;
649 pair = (char *) alloca(strlen(to) + strlen(from) + 2);
654 for (i = 0; i < ARRAY_SIZE (conversion_tab); i++)
655 if (!strcasecmp (pair, conversion_tab[i].pair))
657 ret.func = conversion_tab[i].func;
658 ret.cd = conversion_tab[i].fake_cd;
663 /* No custom converter - try iconv. */
666 ret.func = convert_using_iconv;
667 ret.cd = iconv_open (to, from);
670 if (ret.cd == (iconv_t) -1)
673 cpp_error (pfile, CPP_DL_ERROR, /* FIXME should be DL_SORRY */
674 "conversion from %s to %s not supported by iconv",
677 cpp_errno (pfile, CPP_DL_ERROR, "iconv_open");
679 ret.func = convert_no_conversion;
684 cpp_error (pfile, CPP_DL_ERROR, /* FIXME: should be DL_SORRY */
685 "no iconv implementation, cannot convert from %s to %s",
687 ret.func = convert_no_conversion;
688 ret.cd = (iconv_t) -1;
694 /* If charset conversion is requested, initialize iconv(3) descriptors
695 for conversion from the source character set to the execution
696 character sets. If iconv is not present in the C library, and
697 conversion is requested, issue an error. */
700 cpp_init_iconv (cpp_reader *pfile)
702 const char *ncset = CPP_OPTION (pfile, narrow_charset);
703 const char *wcset = CPP_OPTION (pfile, wide_charset);
704 const char *default_wcset;
706 bool be = CPP_OPTION (pfile, bytes_big_endian);
708 if (CPP_OPTION (pfile, wchar_precision) >= 32)
709 default_wcset = be ? "UTF-32BE" : "UTF-32LE";
710 else if (CPP_OPTION (pfile, wchar_precision) >= 16)
711 default_wcset = be ? "UTF-16BE" : "UTF-16LE";
713 /* This effectively means that wide strings are not supported,
714 so don't do any conversion at all. */
715 default_wcset = SOURCE_CHARSET;
718 ncset = SOURCE_CHARSET;
720 wcset = default_wcset;
722 pfile->narrow_cset_desc = init_iconv_desc (pfile, ncset, SOURCE_CHARSET);
723 pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
724 pfile->char16_cset_desc = init_iconv_desc (pfile,
725 be ? "UTF-16BE" : "UTF-16LE",
727 pfile->char16_cset_desc.width = 16;
728 pfile->char32_cset_desc = init_iconv_desc (pfile,
729 be ? "UTF-32BE" : "UTF-32LE",
731 pfile->char32_cset_desc.width = 32;
732 pfile->wide_cset_desc = init_iconv_desc (pfile, wcset, SOURCE_CHARSET);
733 pfile->wide_cset_desc.width = CPP_OPTION (pfile, wchar_precision);
736 /* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
738 _cpp_destroy_iconv (cpp_reader *pfile)
742 if (pfile->narrow_cset_desc.func == convert_using_iconv)
743 iconv_close (pfile->narrow_cset_desc.cd);
744 if (pfile->wide_cset_desc.func == convert_using_iconv)
745 iconv_close (pfile->wide_cset_desc.cd);
749 /* Utility routine for use by a full compiler. C is a character taken
750 from the *basic* source character set, encoded in the host's
751 execution encoding. Convert it to (the target's) execution
752 encoding, and return that value.
754 Issues an internal error if C's representation in the narrow
755 execution character set fails to be a single-byte value (C99
756 5.2.1p3: "The representation of each member of the source and
757 execution character sets shall fit in a byte.") May also issue an
758 internal error if C fails to be a member of the basic source
759 character set (testing this exactly is too hard, especially when
760 the host character set is EBCDIC). */
762 cpp_host_to_exec_charset (cpp_reader *pfile, cppchar_t c)
765 struct _cpp_strbuf tbuf;
767 /* This test is merely an approximation, but it suffices to catch
768 the most important thing, which is that we don't get handed a
769 character outside the unibyte range of the host character set. */
770 if (c > LAST_POSSIBLY_BASIC_SOURCE_CHAR)
772 cpp_error (pfile, CPP_DL_ICE,
773 "character 0x%lx is not in the basic source character set\n",
778 /* Being a character in the unibyte range of the host character set,
779 we can safely splat it into a one-byte buffer and trust that that
780 is a well-formed string. */
783 /* This should never need to reallocate, but just in case... */
785 tbuf.text = XNEWVEC (uchar, tbuf.asize);
788 if (!APPLY_CONVERSION (pfile->narrow_cset_desc, sbuf, 1, &tbuf))
790 cpp_errno (pfile, CPP_DL_ICE, "converting to execution character set");
795 cpp_error (pfile, CPP_DL_ICE,
796 "character 0x%lx is not unibyte in execution character set",
807 /* Utility routine that computes a mask of the form 0000...111... with
810 width_to_mask (size_t width)
812 width = MIN (width, BITS_PER_CPPCHAR_T);
813 if (width >= CHAR_BIT * sizeof (size_t))
816 return ((size_t) 1 << width) - 1;
819 /* A large table of unicode character information. */
821 /* Valid in a C99 identifier? */
823 /* Valid in a C99 identifier, but not as the first character? */
825 /* Valid in a C++ identifier? */
827 /* NFC representation is not valid in an identifier? */
829 /* Might be valid NFC form? */
831 /* Might be valid NFKC form? */
833 /* Certain preceding characters might make it not valid NFC/NKFC form? */
837 static const struct {
838 /* Bitmap of flags above. */
840 /* Combining class of the character. */
841 unsigned char combine;
842 /* Last character in the range described by this entry. */
848 /* Returns 1 if C is valid in an identifier, 2 if C is valid except at
849 the start of an identifier, and 0 if C is not valid in an
850 identifier. We assume C has already gone through the checks of
851 _cpp_valid_ucn. Also update NST for C if returning nonzero. The
852 algorithm is a simple binary search on the table defined in
856 ucn_valid_in_identifier (cpp_reader *pfile, cppchar_t c,
857 struct normalize_state *nst)
865 mx = ARRAY_SIZE (ucnranges) - 1;
869 if (c <= ucnranges[md].end)
875 /* When -pedantic, we require the character to have been listed by
876 the standard for the current language. Otherwise, we accept the
877 union of the acceptable sets for C++98 and C99. */
878 if (! (ucnranges[mn].flags & (C99 | CXX)))
881 if (CPP_PEDANTIC (pfile)
882 && ((CPP_OPTION (pfile, c99) && !(ucnranges[mn].flags & C99))
883 || (CPP_OPTION (pfile, cplusplus)
884 && !(ucnranges[mn].flags & CXX))))
888 if (ucnranges[mn].combine != 0 && ucnranges[mn].combine < nst->prev_class)
889 nst->level = normalized_none;
890 else if (ucnranges[mn].flags & CTX)
893 cppchar_t p = nst->previous;
895 /* Easy cases from Bengali, Oriya, Tamil, Jannada, and Malayalam. */
897 safe = p != 0x09C7; /* Use 09CB instead of 09C7 09BE. */
898 else if (c == 0x0B3E)
899 safe = p != 0x0B47; /* Use 0B4B instead of 0B47 0B3E. */
900 else if (c == 0x0BBE)
901 safe = p != 0x0BC6 && p != 0x0BC7; /* Use 0BCA/0BCB instead. */
902 else if (c == 0x0CC2)
903 safe = p != 0x0CC6; /* Use 0CCA instead of 0CC6 0CC2. */
904 else if (c == 0x0D3E)
905 safe = p != 0x0D46 && p != 0x0D47; /* Use 0D4A/0D4B instead. */
906 /* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
907 and are combined algorithmically from a sequence of the form
908 1100-1112 1161-1175 11A8-11C2
909 (if the third is not present, it is treated as 11A7, which is not
910 really a valid character).
911 Unfortunately, C99 allows (only) the NFC form, but C++ allows
912 only the combining characters. */
913 else if (c >= 0x1161 && c <= 0x1175)
914 safe = p < 0x1100 || p > 0x1112;
915 else if (c >= 0x11A8 && c <= 0x11C2)
916 safe = (p < 0xAC00 || p > 0xD7A3 || (p - 0xAC00) % 28 != 0);
919 /* Uh-oh, someone updated ucnid.h without updating this code. */
920 cpp_error (pfile, CPP_DL_ICE, "Character %x might not be NFKC", c);
923 if (!safe && c < 0x1161)
924 nst->level = normalized_none;
926 nst->level = MAX (nst->level, normalized_identifier_C);
928 else if (ucnranges[mn].flags & NKC)
930 else if (ucnranges[mn].flags & NFC)
931 nst->level = MAX (nst->level, normalized_C);
932 else if (ucnranges[mn].flags & CID)
933 nst->level = MAX (nst->level, normalized_identifier_C);
935 nst->level = normalized_none;
937 nst->prev_class = ucnranges[mn].combine;
939 /* In C99, UCN digits may not begin identifiers. */
940 if (CPP_OPTION (pfile, c99) && (ucnranges[mn].flags & DIG))
946 /* [lex.charset]: The character designated by the universal character
947 name \UNNNNNNNN is that character whose character short name in
948 ISO/IEC 10646 is NNNNNNNN; the character designated by the
949 universal character name \uNNNN is that character whose character
950 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
951 for a universal character name corresponds to a surrogate code point
952 (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
953 Additionally, if the hexadecimal value for a universal-character-name
954 outside a character or string literal corresponds to a control character
955 (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
956 character in the basic source character set, the program is ill-formed.
958 C99 6.4.3: A universal character name shall not specify a character
959 whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
960 or 0060 (`), nor one in the range D800 through DFFF inclusive.
962 *PSTR must be preceded by "\u" or "\U"; it is assumed that the
963 buffer end is delimited by a non-hex digit. Returns zero if the
964 UCN has not been consumed.
966 Otherwise the nonzero value of the UCN, whether valid or invalid,
967 is returned. Diagnostics are emitted for invalid values. PSTR
968 is updated to point one beyond the UCN, or to the syntactically
971 IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
972 an identifier, or 2 otherwise. */
975 _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
976 const uchar *limit, int identifier_pos,
977 struct normalize_state *nst)
981 const uchar *str = *pstr;
982 const uchar *base = str - 2;
984 if (!CPP_OPTION (pfile, cplusplus) && !CPP_OPTION (pfile, c99))
985 cpp_error (pfile, CPP_DL_WARNING,
986 "universal character names are only valid in C++ and C99");
987 else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0)
988 cpp_error (pfile, CPP_DL_WARNING,
989 "the meaning of '\\%c' is different in traditional C",
994 else if (str[-1] == 'U')
998 cpp_error (pfile, CPP_DL_ICE, "In _cpp_valid_ucn but not a UCN");
1009 result = (result << 4) + hex_value (c);
1011 while (--length && str < limit);
1013 /* Partial UCNs are not valid in strings, but decompose into
1014 multiple tokens in identifiers, so we can't give a helpful
1015 error message in that case. */
1016 if (length && identifier_pos)
1022 cpp_error (pfile, CPP_DL_ERROR,
1023 "incomplete universal character name %.*s",
1024 (int) (str - base), base);
1027 /* The C99 standard permits $, @ and ` to be specified as UCNs. We use
1028 hex escapes so that this also works with EBCDIC hosts.
1029 C++0x permits everything below 0xa0 within literals;
1030 ucn_valid_in_identifier will complain about identifiers. */
1031 else if ((result < 0xa0
1032 && !CPP_OPTION (pfile, cplusplus)
1033 && (result != 0x24 && result != 0x40 && result != 0x60))
1034 || (result & 0x80000000)
1035 || (result >= 0xD800 && result <= 0xDFFF))
1037 cpp_error (pfile, CPP_DL_ERROR,
1038 "%.*s is not a valid universal character",
1039 (int) (str - base), base);
1042 else if (identifier_pos && result == 0x24
1043 && CPP_OPTION (pfile, dollars_in_ident))
1045 if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping)
1047 CPP_OPTION (pfile, warn_dollars) = 0;
1048 cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number");
1050 NORMALIZE_STATE_UPDATE_IDNUM (nst);
1052 else if (identifier_pos)
1054 int validity = ucn_valid_in_identifier (pfile, result, nst);
1057 cpp_error (pfile, CPP_DL_ERROR,
1058 "universal character %.*s is not valid in an identifier",
1059 (int) (str - base), base);
1060 else if (validity == 2 && identifier_pos == 1)
1061 cpp_error (pfile, CPP_DL_ERROR,
1062 "universal character %.*s is not valid at the start of an identifier",
1063 (int) (str - base), base);
1072 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
1073 it to the execution character set and write the result into TBUF.
1074 An advanced pointer is returned. Issues all relevant diagnostics. */
1075 static const uchar *
1076 convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
1077 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1082 size_t bytesleft = 6;
1084 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
1086 from++; /* Skip u/U. */
1087 ucn = _cpp_valid_ucn (pfile, &from, limit, 0, &nst);
1089 rval = one_cppchar_to_utf8 (ucn, &bufp, &bytesleft);
1093 cpp_errno (pfile, CPP_DL_ERROR,
1094 "converting UCN to source character set");
1096 else if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
1097 cpp_errno (pfile, CPP_DL_ERROR,
1098 "converting UCN to execution character set");
1103 /* Subroutine of convert_hex and convert_oct. N is the representation
1104 in the execution character set of a numeric escape; write it into the
1105 string buffer TBUF and update the end-of-string pointer therein. WIDE
1106 is true if it's a wide string that's being assembled in TBUF. This
1107 function issues no diagnostics and never fails. */
1109 emit_numeric_escape (cpp_reader *pfile, cppchar_t n,
1110 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1112 size_t width = cvt.width;
1114 if (width != CPP_OPTION (pfile, char_precision))
1116 /* We have to render this into the target byte order, which may not
1117 be our byte order. */
1118 bool bigend = CPP_OPTION (pfile, bytes_big_endian);
1119 size_t cwidth = CPP_OPTION (pfile, char_precision);
1120 size_t cmask = width_to_mask (cwidth);
1121 size_t nbwc = width / cwidth;
1123 size_t off = tbuf->len;
1126 if (tbuf->len + nbwc > tbuf->asize)
1128 tbuf->asize += OUTBUF_BLOCK_SIZE;
1129 tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
1132 for (i = 0; i < nbwc; i++)
1136 tbuf->text[off + (bigend ? nbwc - i - 1 : i)] = c;
1142 /* Note: this code does not handle the case where the target
1143 and host have a different number of bits in a byte. */
1144 if (tbuf->len + 1 > tbuf->asize)
1146 tbuf->asize += OUTBUF_BLOCK_SIZE;
1147 tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
1149 tbuf->text[tbuf->len++] = n;
1153 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
1154 character set and write it into the string buffer TBUF. Returns an
1155 advanced pointer, and issues diagnostics as necessary.
1156 No character set translation occurs; this routine always produces the
1157 execution-set character with numeric value equal to the given hex
1158 number. You can, e.g. generate surrogate pairs this way. */
1159 static const uchar *
1160 convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
1161 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1163 cppchar_t c, n = 0, overflow = 0;
1164 int digits_found = 0;
1165 size_t width = cvt.width;
1166 size_t mask = width_to_mask (width);
1168 if (CPP_WTRADITIONAL (pfile))
1169 cpp_error (pfile, CPP_DL_WARNING,
1170 "the meaning of '\\x' is different in traditional C");
1172 from++; /* Skip 'x'. */
1173 while (from < limit)
1179 overflow |= n ^ (n << 4 >> 4);
1180 n = (n << 4) + hex_value (c);
1186 cpp_error (pfile, CPP_DL_ERROR,
1187 "\\x used with no following hex digits");
1191 if (overflow | (n != (n & mask)))
1193 cpp_error (pfile, CPP_DL_PEDWARN,
1194 "hex escape sequence out of range");
1198 emit_numeric_escape (pfile, n, tbuf, cvt);
1203 /* Convert an octal escape, pointed to by FROM, to the execution
1204 character set and write it into the string buffer TBUF. Returns an
1205 advanced pointer, and issues diagnostics as necessary.
1206 No character set translation occurs; this routine always produces the
1207 execution-set character with numeric value equal to the given octal
1209 static const uchar *
1210 convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
1211 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1215 size_t width = cvt.width;
1216 size_t mask = width_to_mask (width);
1217 bool overflow = false;
1219 while (from < limit && count++ < 3)
1222 if (c < '0' || c > '7')
1225 overflow |= n ^ (n << 3 >> 3);
1226 n = (n << 3) + c - '0';
1229 if (n != (n & mask))
1231 cpp_error (pfile, CPP_DL_PEDWARN,
1232 "octal escape sequence out of range");
1236 emit_numeric_escape (pfile, n, tbuf, cvt);
1241 /* Convert an escape sequence (pointed to by FROM) to its value on
1242 the target, and to the execution character set. Do not scan past
1243 LIMIT. Write the converted value into TBUF. Returns an advanced
1244 pointer. Handles all relevant diagnostics. */
1245 static const uchar *
1246 convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
1247 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1249 /* Values of \a \b \e \f \n \r \t \v respectively. */
1250 #if HOST_CHARSET == HOST_CHARSET_ASCII
1251 static const uchar charconsts[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
1252 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
1253 static const uchar charconsts[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
1255 #error "unknown host character set"
1263 /* UCNs, hex escapes, and octal escapes are processed separately. */
1265 return convert_ucn (pfile, from, limit, tbuf, cvt);
1268 return convert_hex (pfile, from, limit, tbuf, cvt);
1271 case '0': case '1': case '2': case '3':
1272 case '4': case '5': case '6': case '7':
1273 return convert_oct (pfile, from, limit, tbuf, cvt);
1275 /* Various letter escapes. Get the appropriate host-charset
1277 case '\\': case '\'': case '"': case '?': break;
1279 case '(': case '{': case '[': case '%':
1280 /* '\(', etc, can be used at the beginning of a line in a long
1281 string split onto multiple lines with \-newline, to prevent
1282 Emacs or other text editors from getting confused. '\%' can
1283 be used to prevent SCCS from mangling printf format strings. */
1284 if (CPP_PEDANTIC (pfile))
1288 case 'b': c = charconsts[1]; break;
1289 case 'f': c = charconsts[3]; break;
1290 case 'n': c = charconsts[4]; break;
1291 case 'r': c = charconsts[5]; break;
1292 case 't': c = charconsts[6]; break;
1293 case 'v': c = charconsts[7]; break;
1296 if (CPP_WTRADITIONAL (pfile))
1297 cpp_error (pfile, CPP_DL_WARNING,
1298 "the meaning of '\\a' is different in traditional C");
1303 if (CPP_PEDANTIC (pfile))
1304 cpp_error (pfile, CPP_DL_PEDWARN,
1305 "non-ISO-standard escape sequence, '\\%c'", (int) c);
1312 cpp_error (pfile, CPP_DL_PEDWARN,
1313 "unknown escape sequence: '\\%c'", (int) c);
1316 /* diagnostic.c does not support "%03o". When it does, this
1317 code can use %03o directly in the diagnostic again. */
1319 sprintf(buf, "%03o", (int) c);
1320 cpp_error (pfile, CPP_DL_PEDWARN,
1321 "unknown escape sequence: '\\%s'", buf);
1325 /* Now convert what we have to the execution character set. */
1326 if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
1327 cpp_errno (pfile, CPP_DL_ERROR,
1328 "converting escape sequence to execution character set");
1333 /* TYPE is a token type. The return value is the conversion needed to
1334 convert from source to execution character set for the given type. */
1335 static struct cset_converter
1336 converter_for_type (cpp_reader *pfile, enum cpp_ttype type)
1341 return pfile->narrow_cset_desc;
1344 return pfile->char16_cset_desc;
1347 return pfile->char32_cset_desc;
1350 return pfile->wide_cset_desc;
1354 /* FROM is an array of cpp_string structures of length COUNT. These
1355 are to be converted from the source to the execution character set,
1356 escape sequences translated, and finally all are to be
1357 concatenated. WIDE indicates whether or not to produce a wide
1358 string. The result is written into TO. Returns true for success,
1359 false for failure. */
1361 cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
1362 cpp_string *to, enum cpp_ttype type)
1364 struct _cpp_strbuf tbuf;
1365 const uchar *p, *base, *limit;
1367 struct cset_converter cvt = converter_for_type (pfile, type);
1369 tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
1370 tbuf.text = XNEWVEC (uchar, tbuf.asize);
1373 for (i = 0; i < count; i++)
1376 if (*p == 'L' || *p == 'u' || *p == 'U') p++;
1377 p++; /* Skip leading quote. */
1378 limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */
1383 while (p < limit && *p != '\\')
1387 /* We have a run of normal characters; these can be fed
1388 directly to convert_cset. */
1389 if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
1395 p = convert_escape (pfile, p + 1, limit, &tbuf, cvt);
1398 /* NUL-terminate the 'to' buffer and translate it to a cpp_string
1400 emit_numeric_escape (pfile, 0, &tbuf, cvt);
1401 tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
1402 to->text = tbuf.text;
1407 cpp_errno (pfile, CPP_DL_ERROR, "converting to execution character set");
1412 /* Subroutine of do_line and do_linemarker. Convert escape sequences
1413 in a string, but do not perform character set conversion. */
1415 cpp_interpret_string_notranslate (cpp_reader *pfile, const cpp_string *from,
1416 size_t count, cpp_string *to,
1417 enum cpp_ttype type ATTRIBUTE_UNUSED)
1419 struct cset_converter save_narrow_cset_desc = pfile->narrow_cset_desc;
1422 pfile->narrow_cset_desc.func = convert_no_conversion;
1423 pfile->narrow_cset_desc.cd = (iconv_t) -1;
1424 pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
1426 retval = cpp_interpret_string (pfile, from, count, to, CPP_STRING);
1428 pfile->narrow_cset_desc = save_narrow_cset_desc;
1433 /* Subroutine of cpp_interpret_charconst which performs the conversion
1434 to a number, for narrow strings. STR is the string structure returned
1435 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1436 cpp_interpret_charconst. */
1438 narrow_str_to_charconst (cpp_reader *pfile, cpp_string str,
1439 unsigned int *pchars_seen, int *unsignedp)
1441 size_t width = CPP_OPTION (pfile, char_precision);
1442 size_t max_chars = CPP_OPTION (pfile, int_precision) / width;
1443 size_t mask = width_to_mask (width);
1445 cppchar_t result, c;
1448 /* The value of a multi-character character constant, or a
1449 single-character character constant whose representation in the
1450 execution character set is more than one byte long, is
1451 implementation defined. This implementation defines it to be the
1452 number formed by interpreting the byte sequence in memory as a
1453 big-endian binary number. If overflow occurs, the high bytes are
1454 lost, and a warning is issued.
1456 We don't want to process the NUL terminator handed back by
1457 cpp_interpret_string. */
1459 for (i = 0; i < str.len - 1; i++)
1461 c = str.text[i] & mask;
1462 if (width < BITS_PER_CPPCHAR_T)
1463 result = (result << width) | c;
1471 cpp_error (pfile, CPP_DL_WARNING,
1472 "character constant too long for its type");
1474 else if (i > 1 && CPP_OPTION (pfile, warn_multichar))
1475 cpp_error (pfile, CPP_DL_WARNING, "multi-character character constant");
1477 /* Multichar constants are of type int and therefore signed. */
1481 unsigned_p = CPP_OPTION (pfile, unsigned_char);
1483 /* Truncate the constant to its natural width, and simultaneously
1484 sign- or zero-extend to the full width of cppchar_t.
1485 For single-character constants, the value is WIDTH bits wide.
1486 For multi-character constants, the value is INT_PRECISION bits wide. */
1488 width = CPP_OPTION (pfile, int_precision);
1489 if (width < BITS_PER_CPPCHAR_T)
1491 mask = ((cppchar_t) 1 << width) - 1;
1492 if (unsigned_p || !(result & (1 << (width - 1))))
1498 *unsignedp = unsigned_p;
1502 /* Subroutine of cpp_interpret_charconst which performs the conversion
1503 to a number, for wide strings. STR is the string structure returned
1504 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1505 cpp_interpret_charconst. TYPE is the token type. */
1507 wide_str_to_charconst (cpp_reader *pfile, cpp_string str,
1508 unsigned int *pchars_seen, int *unsignedp,
1509 enum cpp_ttype type)
1511 bool bigend = CPP_OPTION (pfile, bytes_big_endian);
1512 size_t width = converter_for_type (pfile, type).width;
1513 size_t cwidth = CPP_OPTION (pfile, char_precision);
1514 size_t mask = width_to_mask (width);
1515 size_t cmask = width_to_mask (cwidth);
1516 size_t nbwc = width / cwidth;
1518 cppchar_t result = 0, c;
1520 /* This is finicky because the string is in the target's byte order,
1521 which may not be our byte order. Only the last character, ignoring
1522 the NUL terminator, is relevant. */
1523 off = str.len - (nbwc * 2);
1525 for (i = 0; i < nbwc; i++)
1527 c = bigend ? str.text[off + i] : str.text[off + nbwc - i - 1];
1528 result = (result << cwidth) | (c & cmask);
1531 /* Wide character constants have type wchar_t, and a single
1532 character exactly fills a wchar_t, so a multi-character wide
1533 character constant is guaranteed to overflow. */
1534 if (str.len > nbwc * 2)
1535 cpp_error (pfile, CPP_DL_WARNING,
1536 "character constant too long for its type");
1538 /* Truncate the constant to its natural width, and simultaneously
1539 sign- or zero-extend to the full width of cppchar_t. */
1540 if (width < BITS_PER_CPPCHAR_T)
1542 if (type == CPP_CHAR16 || type == CPP_CHAR32
1543 || CPP_OPTION (pfile, unsigned_wchar)
1544 || !(result & (1 << (width - 1))))
1550 if (type == CPP_CHAR16 || type == CPP_CHAR32
1551 || CPP_OPTION (pfile, unsigned_wchar))
1560 /* Interpret a (possibly wide) character constant in TOKEN.
1561 PCHARS_SEEN points to a variable that is filled in with the number
1562 of characters seen, and UNSIGNEDP to a variable that indicates
1563 whether the result has signed type. */
1565 cpp_interpret_charconst (cpp_reader *pfile, const cpp_token *token,
1566 unsigned int *pchars_seen, int *unsignedp)
1568 cpp_string str = { 0, 0 };
1569 bool wide = (token->type != CPP_CHAR);
1572 /* an empty constant will appear as L'', u'', U'' or '' */
1573 if (token->val.str.len == (size_t) (2 + wide))
1575 cpp_error (pfile, CPP_DL_ERROR, "empty character constant");
1578 else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, token->type))
1582 result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp,
1585 result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp);
1587 if (str.text != token->val.str.text)
1588 free ((void *)str.text);
1593 /* Convert an identifier denoted by ID and LEN, which might contain
1594 UCN escapes, to the source character set, either UTF-8 or
1595 UTF-EBCDIC. Assumes that the identifier is actually a valid identifier. */
1597 _cpp_interpret_identifier (cpp_reader *pfile, const uchar *id, size_t len)
1599 /* It turns out that a UCN escape always turns into fewer characters
1600 than the escape itself, so we can allocate a temporary in advance. */
1601 uchar * buf = (uchar *) alloca (len + 1);
1605 for (idp = 0; idp < len; idp++)
1606 if (id[idp] != '\\')
1610 unsigned length = id[idp+1] == 'u' ? 4 : 8;
1611 cppchar_t value = 0;
1612 size_t bufleft = len - (bufp - buf);
1616 while (length && idp < len && ISXDIGIT (id[idp]))
1618 value = (value << 4) + hex_value (id[idp]);
1624 /* Special case for EBCDIC: if the identifier contains
1625 a '$' specified using a UCN, translate it to EBCDIC. */
1632 rval = one_cppchar_to_utf8 (value, &bufp, &bufleft);
1636 cpp_errno (pfile, CPP_DL_ERROR,
1637 "converting UCN to source character set");
1642 return CPP_HASHNODE (ht_lookup (pfile->hash_table,
1643 buf, bufp - buf, HT_ALLOC));
1646 /* Convert an input buffer (containing the complete contents of one
1647 source file) from INPUT_CHARSET to the source character set. INPUT
1648 points to the input buffer, SIZE is its allocated size, and LEN is
1649 the length of the meaningful data within the buffer. The
1650 translated buffer is returned, *ST_SIZE is set to the length of
1651 the meaningful data within the translated buffer, and *BUFFER_START
1652 is set to the start of the returned buffer. *BUFFER_START may
1653 differ from the return value in the case of a BOM or other ignored
1656 INPUT is expected to have been allocated with xmalloc. This
1657 function will either set *BUFFER_START to INPUT, or free it and set
1658 *BUFFER_START to a pointer to another xmalloc-allocated block of
1661 _cpp_convert_input (cpp_reader *pfile, const char *input_charset,
1662 uchar *input, size_t size, size_t len,
1663 const unsigned char **buffer_start, off_t *st_size)
1665 struct cset_converter input_cset;
1666 struct _cpp_strbuf to;
1667 unsigned char *buffer;
1669 input_cset = init_iconv_desc (pfile, SOURCE_CHARSET, input_charset);
1670 if (input_cset.func == convert_no_conversion)
1678 to.asize = MAX (65536, len);
1679 to.text = XNEWVEC (uchar, to.asize);
1682 if (!APPLY_CONVERSION (input_cset, input, len, &to))
1683 cpp_error (pfile, CPP_DL_ERROR,
1684 "failure to convert %s to %s",
1685 CPP_OPTION (pfile, input_charset), SOURCE_CHARSET);
1690 /* Clean up the mess. */
1691 if (input_cset.func == convert_using_iconv)
1692 iconv_close (input_cset.cd);
1694 /* Resize buffer if we allocated substantially too much, or if we
1695 haven't enough space for the \n-terminator. */
1696 if (to.len + 4096 < to.asize || to.len >= to.asize)
1697 to.text = XRESIZEVEC (uchar, to.text, to.len + 1);
1699 /* If the file is using old-school Mac line endings (\r only),
1700 terminate with another \r, not an \n, so that we do not mistake
1701 the \r\n sequence for a single DOS line ending and erroneously
1702 issue the "No newline at end of file" diagnostic. */
1703 if (to.len && to.text[to.len - 1] == '\r')
1704 to.text[to.len] = '\r';
1706 to.text[to.len] = '\n';
1710 #if HOST_CHARSET == HOST_CHARSET_ASCII
1711 /* The HOST_CHARSET test just above ensures that the source charset
1712 is UTF-8. So, ignore a UTF-8 BOM if we see one. Note that
1713 glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
1714 BOM -- however, even if it did, we would still need this code due
1715 to the 'convert_no_conversion' case. */
1716 if (to.len >= 3 && to.text[0] == 0xef && to.text[1] == 0xbb
1717 && to.text[2] == 0xbf)
1724 *buffer_start = to.text;
1728 /* Decide on the default encoding to assume for input files. */
1730 _cpp_default_encoding (void)
1732 const char *current_encoding = NULL;
1734 /* We disable this because the default codeset is 7-bit ASCII on
1735 most platforms, and this causes conversion failures on every
1736 file in GCC that happens to have one of the upper 128 characters
1737 in it -- most likely, as part of the name of a contributor.
1738 We should definitely recognize in-band markers of file encoding,
1740 - the appropriate Unicode byte-order mark (FE FF) to recognize
1741 UTF16 and UCS4 (in both big-endian and little-endian flavors)
1743 - a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to
1744 distinguish ASCII and EBCDIC.
1745 - now we can parse something like "#pragma GCC encoding <xyz>
1746 on the first line, or even Emacs/VIM's mode line tags (there's
1747 a problem here in that VIM uses the last line, and Emacs has
1748 its more elaborate "local variables" convention).
1749 - investigate whether Java has another common convention, which
1750 would be friendly to support.
1751 (Zack Weinberg and Paolo Bonzini, May 20th 2004) */
1752 #if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0
1753 setlocale (LC_CTYPE, "");
1754 current_encoding = nl_langinfo (CODESET);
1756 if (current_encoding == NULL || *current_encoding == '\0')
1757 current_encoding = SOURCE_CHARSET;
1759 return current_encoding;