X-Git-Url: http://git.sourceforge.jp/view?a=blobdiff_plain;f=libcpp%2Fcharset.c;h=e743b1e277fcd412972bde2d70871697075ca028;hb=941f2388c6654af085ec92de505bc6b53d90b5fa;hp=f382d7f57fcad445e6872a79f5a25b8195c4031b;hpb=720aca9227005bb5d8eadd8bd95d630a28568180;p=pf3gnuchains%2Fgcc-fork.git diff --git a/libcpp/charset.c b/libcpp/charset.c index f382d7f57fc..e743b1e277f 100644 --- a/libcpp/charset.c +++ b/libcpp/charset.c @@ -1,12 +1,12 @@ /* CPP Library - charsets - Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004 + Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2006, 2008, 2009 Free Software Foundation, Inc. Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the -Free Software Foundation; either version 2, or (at your option) any +Free Software Foundation; either version 3, or (at your option) any later version. This program is distributed in the hope that it will be useful, @@ -15,8 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ +along with this program; see the file COPYING3. If not see +. */ #include "config.h" #include "system.h" @@ -548,6 +548,15 @@ convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED, /* And this one uses the system iconv primitive. It's a little different, since iconv's interface is a little different. */ #if HAVE_ICONV + +#define CONVERT_ICONV_GROW_BUFFER \ + do { \ + outbytesleft += OUTBUF_BLOCK_SIZE; \ + to->asize += OUTBUF_BLOCK_SIZE; \ + to->text = XRESIZEVEC (uchar, to->text, to->asize); \ + outbuf = (char *)to->text + to->asize - outbytesleft; \ + } while (0) + static bool convert_using_iconv (iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to) @@ -570,16 +579,24 @@ convert_using_iconv (iconv_t cd, const uchar *from, size_t flen, iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); if (__builtin_expect (inbytesleft == 0, 1)) { + /* Close out any shift states, returning to the initial state. */ + if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1) + { + if (errno != E2BIG) + return false; + + CONVERT_ICONV_GROW_BUFFER; + if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1) + return false; + } + to->len = to->asize - outbytesleft; return true; } if (errno != E2BIG) return false; - outbytesleft += OUTBUF_BLOCK_SIZE; - to->asize += OUTBUF_BLOCK_SIZE; - to->text = XRESIZEVEC (uchar, to->text, to->asize); - outbuf = (char *)to->text + to->asize - outbytesleft; + CONVERT_ICONV_GROW_BUFFER; } } #else @@ -625,6 +642,7 @@ init_iconv_desc (cpp_reader *pfile, const char *to, const char *from) { ret.func = convert_no_conversion; ret.cd = (iconv_t) -1; + ret.width = -1; return ret; } @@ -638,6 +656,7 @@ init_iconv_desc (cpp_reader *pfile, const char *to, const char *from) { ret.func = conversion_tab[i].func; ret.cd = conversion_tab[i].fake_cd; + ret.width = -1; return ret; } @@ -646,6 +665,7 @@ init_iconv_desc (cpp_reader *pfile, const char *to, const char *from) { ret.func = convert_using_iconv; ret.cd = iconv_open (to, from); + ret.width = -1; if (ret.cd == (iconv_t) -1) { @@ -666,6 +686,7 @@ init_iconv_desc (cpp_reader *pfile, const char *to, const char *from) from, to); ret.func = convert_no_conversion; ret.cd = (iconv_t) -1; + ret.width = -1; } return ret; } @@ -699,7 +720,17 @@ cpp_init_iconv (cpp_reader *pfile) wcset = default_wcset; pfile->narrow_cset_desc = init_iconv_desc (pfile, ncset, SOURCE_CHARSET); + pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision); + pfile->char16_cset_desc = init_iconv_desc (pfile, + be ? "UTF-16BE" : "UTF-16LE", + SOURCE_CHARSET); + pfile->char16_cset_desc.width = 16; + pfile->char32_cset_desc = init_iconv_desc (pfile, + be ? "UTF-32BE" : "UTF-32LE", + SOURCE_CHARSET); + pfile->char32_cset_desc.width = 32; pfile->wide_cset_desc = init_iconv_desc (pfile, wcset, SOURCE_CHARSET); + pfile->wide_cset_desc.width = CPP_OPTION (pfile, wchar_precision); } /* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */ @@ -1034,15 +1065,13 @@ _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr, An advanced pointer is returned. Issues all relevant diagnostics. */ static const uchar * convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit, - struct _cpp_strbuf *tbuf, bool wide) + struct _cpp_strbuf *tbuf, struct cset_converter cvt) { cppchar_t ucn; uchar buf[6]; uchar *bufp = buf; size_t bytesleft = 6; int rval; - struct cset_converter cvt - = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc; struct normalize_state nst = INITIAL_NORMALIZE_STATE; from++; /* Skip u/U. */ @@ -1069,14 +1098,15 @@ convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit, function issues no diagnostics and never fails. */ static void emit_numeric_escape (cpp_reader *pfile, cppchar_t n, - struct _cpp_strbuf *tbuf, bool wide) + struct _cpp_strbuf *tbuf, struct cset_converter cvt) { - if (wide) + size_t width = cvt.width; + + if (width != CPP_OPTION (pfile, char_precision)) { /* We have to render this into the target byte order, which may not be our byte order. */ bool bigend = CPP_OPTION (pfile, bytes_big_endian); - size_t width = CPP_OPTION (pfile, wchar_precision); size_t cwidth = CPP_OPTION (pfile, char_precision); size_t cmask = width_to_mask (cwidth); size_t nbwc = width / cwidth; @@ -1119,12 +1149,11 @@ emit_numeric_escape (cpp_reader *pfile, cppchar_t n, number. You can, e.g. generate surrogate pairs this way. */ static const uchar * convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit, - struct _cpp_strbuf *tbuf, bool wide) + struct _cpp_strbuf *tbuf, struct cset_converter cvt) { cppchar_t c, n = 0, overflow = 0; int digits_found = 0; - size_t width = (wide ? CPP_OPTION (pfile, wchar_precision) - : CPP_OPTION (pfile, char_precision)); + size_t width = cvt.width; size_t mask = width_to_mask (width); if (CPP_WTRADITIONAL (pfile)) @@ -1157,7 +1186,7 @@ convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit, n &= mask; } - emit_numeric_escape (pfile, n, tbuf, wide); + emit_numeric_escape (pfile, n, tbuf, cvt); return from; } @@ -1170,12 +1199,11 @@ convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit, number. */ static const uchar * convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit, - struct _cpp_strbuf *tbuf, bool wide) + struct _cpp_strbuf *tbuf, struct cset_converter cvt) { size_t count = 0; cppchar_t c, n = 0; - size_t width = (wide ? CPP_OPTION (pfile, wchar_precision) - : CPP_OPTION (pfile, char_precision)); + size_t width = cvt.width; size_t mask = width_to_mask (width); bool overflow = false; @@ -1196,7 +1224,7 @@ convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit, n &= mask; } - emit_numeric_escape (pfile, n, tbuf, wide); + emit_numeric_escape (pfile, n, tbuf, cvt); return from; } @@ -1207,7 +1235,7 @@ convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit, pointer. Handles all relevant diagnostics. */ static const uchar * convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit, - struct _cpp_strbuf *tbuf, bool wide) + struct _cpp_strbuf *tbuf, struct cset_converter cvt) { /* Values of \a \b \e \f \n \r \t \v respectively. */ #if HOST_CHARSET == HOST_CHARSET_ASCII @@ -1219,23 +1247,21 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit, #endif uchar c; - struct cset_converter cvt - = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc; c = *from; switch (c) { /* UCNs, hex escapes, and octal escapes are processed separately. */ case 'u': case 'U': - return convert_ucn (pfile, from, limit, tbuf, wide); + return convert_ucn (pfile, from, limit, tbuf, cvt); case 'x': - return convert_hex (pfile, from, limit, tbuf, wide); + return convert_hex (pfile, from, limit, tbuf, cvt); break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': - return convert_oct (pfile, from, limit, tbuf, wide); + return convert_oct (pfile, from, limit, tbuf, cvt); /* Various letter escapes. Get the appropriate host-charset value into C. */ @@ -1277,8 +1303,14 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit, cpp_error (pfile, CPP_DL_PEDWARN, "unknown escape sequence '\\%c'", (int) c); else - cpp_error (pfile, CPP_DL_PEDWARN, - "unknown escape sequence: '\\%03o'", (int) c); + { + /* diagnostic.c does not support "%03o". When it does, this + code can use %03o directly in the diagnostic again. */ + char buf[32]; + sprintf(buf, "%03o", (int) c); + cpp_error (pfile, CPP_DL_PEDWARN, + "unknown escape sequence: '\\%s'", buf); + } } /* Now convert what we have to the execution character set. */ @@ -1289,6 +1321,27 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit, return from + 1; } +/* TYPE is a token type. The return value is the conversion needed to + convert from source to execution character set for the given type. */ +static struct cset_converter +converter_for_type (cpp_reader *pfile, enum cpp_ttype type) +{ + switch (type) + { + default: + return pfile->narrow_cset_desc; + case CPP_CHAR16: + case CPP_STRING16: + return pfile->char16_cset_desc; + case CPP_CHAR32: + case CPP_STRING32: + return pfile->char32_cset_desc; + case CPP_WCHAR: + case CPP_WSTRING: + return pfile->wide_cset_desc; + } +} + /* FROM is an array of cpp_string structures of length COUNT. These are to be converted from the source to the execution character set, escape sequences translated, and finally all are to be @@ -1297,13 +1350,12 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit, false for failure. */ bool cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count, - cpp_string *to, bool wide) + cpp_string *to, enum cpp_ttype type) { struct _cpp_strbuf tbuf; const uchar *p, *base, *limit; size_t i; - struct cset_converter cvt - = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc; + struct cset_converter cvt = converter_for_type (pfile, type); tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len); tbuf.text = XNEWVEC (uchar, tbuf.asize); @@ -1312,7 +1364,7 @@ cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count, for (i = 0; i < count; i++) { p = from[i].text; - if (*p == 'L') p++; + if (*p == 'L' || *p == 'u' || *p == 'U') p++; p++; /* Skip leading quote. */ limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */ @@ -1331,12 +1383,12 @@ cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count, if (p == limit) break; - p = convert_escape (pfile, p + 1, limit, &tbuf, wide); + p = convert_escape (pfile, p + 1, limit, &tbuf, cvt); } } /* NUL-terminate the 'to' buffer and translate it to a cpp_string structure. */ - emit_numeric_escape (pfile, 0, &tbuf, wide); + emit_numeric_escape (pfile, 0, &tbuf, cvt); tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len); to->text = tbuf.text; to->len = tbuf.len; @@ -1352,15 +1404,17 @@ cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count, in a string, but do not perform character set conversion. */ bool cpp_interpret_string_notranslate (cpp_reader *pfile, const cpp_string *from, - size_t count, cpp_string *to, bool wide) + size_t count, cpp_string *to, + enum cpp_ttype type ATTRIBUTE_UNUSED) { struct cset_converter save_narrow_cset_desc = pfile->narrow_cset_desc; bool retval; pfile->narrow_cset_desc.func = convert_no_conversion; pfile->narrow_cset_desc.cd = (iconv_t) -1; + pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision); - retval = cpp_interpret_string (pfile, from, count, to, wide); + retval = cpp_interpret_string (pfile, from, count, to, CPP_STRING); pfile->narrow_cset_desc = save_narrow_cset_desc; return retval; @@ -1439,13 +1493,14 @@ narrow_str_to_charconst (cpp_reader *pfile, cpp_string str, /* Subroutine of cpp_interpret_charconst which performs the conversion to a number, for wide strings. STR is the string structure returned by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for - cpp_interpret_charconst. */ + cpp_interpret_charconst. TYPE is the token type. */ static cppchar_t wide_str_to_charconst (cpp_reader *pfile, cpp_string str, - unsigned int *pchars_seen, int *unsignedp) + unsigned int *pchars_seen, int *unsignedp, + enum cpp_ttype type) { bool bigend = CPP_OPTION (pfile, bytes_big_endian); - size_t width = CPP_OPTION (pfile, wchar_precision); + size_t width = converter_for_type (pfile, type).width; size_t cwidth = CPP_OPTION (pfile, char_precision); size_t mask = width_to_mask (width); size_t cmask = width_to_mask (cwidth); @@ -1467,7 +1522,7 @@ wide_str_to_charconst (cpp_reader *pfile, cpp_string str, /* Wide character constants have type wchar_t, and a single character exactly fills a wchar_t, so a multi-character wide character constant is guaranteed to overflow. */ - if (off > 0) + if (str.len > nbwc * 2) cpp_error (pfile, CPP_DL_WARNING, "character constant too long for its type"); @@ -1475,13 +1530,20 @@ wide_str_to_charconst (cpp_reader *pfile, cpp_string str, sign- or zero-extend to the full width of cppchar_t. */ if (width < BITS_PER_CPPCHAR_T) { - if (CPP_OPTION (pfile, unsigned_wchar) || !(result & (1 << (width - 1)))) + if (type == CPP_CHAR16 || type == CPP_CHAR32 + || CPP_OPTION (pfile, unsigned_wchar) + || !(result & (1 << (width - 1)))) result &= mask; else result |= ~mask; } - *unsignedp = CPP_OPTION (pfile, unsigned_wchar); + if (type == CPP_CHAR16 || type == CPP_CHAR32 + || CPP_OPTION (pfile, unsigned_wchar)) + *unsignedp = 1; + else + *unsignedp = 0; + *pchars_seen = 1; return result; } @@ -1495,20 +1557,21 @@ cpp_interpret_charconst (cpp_reader *pfile, const cpp_token *token, unsigned int *pchars_seen, int *unsignedp) { cpp_string str = { 0, 0 }; - bool wide = (token->type == CPP_WCHAR); + bool wide = (token->type != CPP_CHAR); cppchar_t result; - /* an empty constant will appear as L'' or '' */ + /* an empty constant will appear as L'', u'', U'' or '' */ if (token->val.str.len == (size_t) (2 + wide)) { cpp_error (pfile, CPP_DL_ERROR, "empty character constant"); return 0; } - else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, wide)) + else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, token->type)) return 0; if (wide) - result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp); + result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp, + token->type); else result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp); @@ -1575,18 +1638,24 @@ _cpp_interpret_identifier (cpp_reader *pfile, const uchar *id, size_t len) source file) from INPUT_CHARSET to the source character set. INPUT points to the input buffer, SIZE is its allocated size, and LEN is the length of the meaningful data within the buffer. The - translated buffer is returned, and *ST_SIZE is set to the length of - the meaningful data within the translated buffer. - - INPUT is expected to have been allocated with xmalloc. This function - will either return INPUT, or free it and return a pointer to another - xmalloc-allocated block of memory. */ + translated buffer is returned, *ST_SIZE is set to the length of + the meaningful data within the translated buffer, and *BUFFER_START + is set to the start of the returned buffer. *BUFFER_START may + differ from the return value in the case of a BOM or other ignored + marker information. + + INPUT is expected to have been allocated with xmalloc. This + function will either set *BUFFER_START to INPUT, or free it and set + *BUFFER_START to a pointer to another xmalloc-allocated block of + memory. */ uchar * _cpp_convert_input (cpp_reader *pfile, const char *input_charset, - uchar *input, size_t size, size_t len, off_t *st_size) + uchar *input, size_t size, size_t len, + const unsigned char **buffer_start, off_t *st_size) { struct cset_converter input_cset; struct _cpp_strbuf to; + unsigned char *buffer; input_cset = init_iconv_desc (pfile, SOURCE_CHARSET, input_charset); if (input_cset.func == convert_no_conversion) @@ -1622,13 +1691,29 @@ _cpp_convert_input (cpp_reader *pfile, const char *input_charset, terminate with another \r, not an \n, so that we do not mistake the \r\n sequence for a single DOS line ending and erroneously issue the "No newline at end of file" diagnostic. */ - if (to.text[to.len - 1] == '\r') + if (to.len && to.text[to.len - 1] == '\r') to.text[to.len] = '\r'; else to.text[to.len] = '\n'; + buffer = to.text; *st_size = to.len; - return to.text; +#if HOST_CHARSET == HOST_CHARSET_ASCII + /* The HOST_CHARSET test just above ensures that the source charset + is UTF-8. So, ignore a UTF-8 BOM if we see one. Note that + glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a + BOM -- however, even if it did, we would still need this code due + to the 'convert_no_conversion' case. */ + if (to.len >= 3 && to.text[0] == 0xef && to.text[1] == 0xbb + && to.text[2] == 0xbf) + { + *st_size -= 3; + buffer += 3; + } +#endif + + *buffer_start = to.text; + return buffer; } /* Decide on the default encoding to assume for input files. */