/* CPP Library - charsets
- Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2006
+ Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2006, 2008
Free Software Foundation, Inc.
Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
{
ret.func = convert_no_conversion;
ret.cd = (iconv_t) -1;
+ ret.width = -1;
return ret;
}
{
ret.func = conversion_tab[i].func;
ret.cd = conversion_tab[i].fake_cd;
+ ret.width = -1;
return ret;
}
{
ret.func = convert_using_iconv;
ret.cd = iconv_open (to, from);
+ ret.width = -1;
if (ret.cd == (iconv_t) -1)
{
from, to);
ret.func = convert_no_conversion;
ret.cd = (iconv_t) -1;
+ ret.width = -1;
}
return ret;
}
wcset = default_wcset;
pfile->narrow_cset_desc = init_iconv_desc (pfile, ncset, SOURCE_CHARSET);
+ pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
+ pfile->char16_cset_desc = init_iconv_desc (pfile,
+ be ? "UTF-16BE" : "UTF-16LE",
+ SOURCE_CHARSET);
+ pfile->char16_cset_desc.width = 16;
+ pfile->char32_cset_desc = init_iconv_desc (pfile,
+ be ? "UTF-32BE" : "UTF-32LE",
+ SOURCE_CHARSET);
+ pfile->char32_cset_desc.width = 32;
pfile->wide_cset_desc = init_iconv_desc (pfile, wcset, SOURCE_CHARSET);
+ pfile->wide_cset_desc.width = CPP_OPTION (pfile, wchar_precision);
}
/* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
An advanced pointer is returned. Issues all relevant diagnostics. */
static const uchar *
convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt)
{
cppchar_t ucn;
uchar buf[6];
uchar *bufp = buf;
size_t bytesleft = 6;
int rval;
- struct cset_converter cvt
- = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc;
struct normalize_state nst = INITIAL_NORMALIZE_STATE;
from++; /* Skip u/U. */
function issues no diagnostics and never fails. */
static void
emit_numeric_escape (cpp_reader *pfile, cppchar_t n,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt)
{
- if (wide)
+ size_t width = cvt.width;
+
+ if (width != CPP_OPTION (pfile, char_precision))
{
/* We have to render this into the target byte order, which may not
be our byte order. */
bool bigend = CPP_OPTION (pfile, bytes_big_endian);
- size_t width = CPP_OPTION (pfile, wchar_precision);
size_t cwidth = CPP_OPTION (pfile, char_precision);
size_t cmask = width_to_mask (cwidth);
size_t nbwc = width / cwidth;
number. You can, e.g. generate surrogate pairs this way. */
static const uchar *
convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt)
{
cppchar_t c, n = 0, overflow = 0;
int digits_found = 0;
- size_t width = (wide ? CPP_OPTION (pfile, wchar_precision)
- : CPP_OPTION (pfile, char_precision));
+ size_t width = cvt.width;
size_t mask = width_to_mask (width);
if (CPP_WTRADITIONAL (pfile))
n &= mask;
}
- emit_numeric_escape (pfile, n, tbuf, wide);
+ emit_numeric_escape (pfile, n, tbuf, cvt);
return from;
}
number. */
static const uchar *
convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt)
{
size_t count = 0;
cppchar_t c, n = 0;
- size_t width = (wide ? CPP_OPTION (pfile, wchar_precision)
- : CPP_OPTION (pfile, char_precision));
+ size_t width = cvt.width;
size_t mask = width_to_mask (width);
bool overflow = false;
n &= mask;
}
- emit_numeric_escape (pfile, n, tbuf, wide);
+ emit_numeric_escape (pfile, n, tbuf, cvt);
return from;
}
pointer. Handles all relevant diagnostics. */
static const uchar *
convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
- struct _cpp_strbuf *tbuf, bool wide)
+ struct _cpp_strbuf *tbuf, struct cset_converter cvt)
{
/* Values of \a \b \e \f \n \r \t \v respectively. */
#if HOST_CHARSET == HOST_CHARSET_ASCII
#endif
uchar c;
- struct cset_converter cvt
- = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc;
c = *from;
switch (c)
{
/* UCNs, hex escapes, and octal escapes are processed separately. */
case 'u': case 'U':
- return convert_ucn (pfile, from, limit, tbuf, wide);
+ return convert_ucn (pfile, from, limit, tbuf, cvt);
case 'x':
- return convert_hex (pfile, from, limit, tbuf, wide);
+ return convert_hex (pfile, from, limit, tbuf, cvt);
break;
case '0': case '1': case '2': case '3':
case '4': case '5': case '6': case '7':
- return convert_oct (pfile, from, limit, tbuf, wide);
+ return convert_oct (pfile, from, limit, tbuf, cvt);
/* Various letter escapes. Get the appropriate host-charset
value into C. */
return from + 1;
}
\f
+/* TYPE is a token type. The return value is the conversion needed to
+ convert from source to execution character set for the given type. */
+static struct cset_converter
+converter_for_type (cpp_reader *pfile, enum cpp_ttype type)
+{
+ switch (type)
+ {
+ default:
+ return pfile->narrow_cset_desc;
+ case CPP_CHAR16:
+ case CPP_STRING16:
+ return pfile->char16_cset_desc;
+ case CPP_CHAR32:
+ case CPP_STRING32:
+ return pfile->char32_cset_desc;
+ case CPP_WCHAR:
+ case CPP_WSTRING:
+ return pfile->wide_cset_desc;
+ }
+}
+
/* FROM is an array of cpp_string structures of length COUNT. These
are to be converted from the source to the execution character set,
escape sequences translated, and finally all are to be
false for failure. */
bool
cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
- cpp_string *to, bool wide)
+ cpp_string *to, enum cpp_ttype type)
{
struct _cpp_strbuf tbuf;
const uchar *p, *base, *limit;
size_t i;
- struct cset_converter cvt
- = wide ? pfile->wide_cset_desc : pfile->narrow_cset_desc;
+ struct cset_converter cvt = converter_for_type (pfile, type);
tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
tbuf.text = XNEWVEC (uchar, tbuf.asize);
for (i = 0; i < count; i++)
{
p = from[i].text;
- if (*p == 'L') p++;
+ if (*p == 'L' || *p == 'u' || *p == 'U') p++;
p++; /* Skip leading quote. */
limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */
if (p == limit)
break;
- p = convert_escape (pfile, p + 1, limit, &tbuf, wide);
+ p = convert_escape (pfile, p + 1, limit, &tbuf, cvt);
}
}
/* NUL-terminate the 'to' buffer and translate it to a cpp_string
structure. */
- emit_numeric_escape (pfile, 0, &tbuf, wide);
+ emit_numeric_escape (pfile, 0, &tbuf, cvt);
tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
to->text = tbuf.text;
to->len = tbuf.len;
in a string, but do not perform character set conversion. */
bool
cpp_interpret_string_notranslate (cpp_reader *pfile, const cpp_string *from,
- size_t count, cpp_string *to, bool wide)
+ size_t count, cpp_string *to,
+ enum cpp_ttype type ATTRIBUTE_UNUSED)
{
struct cset_converter save_narrow_cset_desc = pfile->narrow_cset_desc;
bool retval;
pfile->narrow_cset_desc.func = convert_no_conversion;
pfile->narrow_cset_desc.cd = (iconv_t) -1;
- retval = cpp_interpret_string (pfile, from, count, to, wide);
+ retval = cpp_interpret_string (pfile, from, count, to, CPP_STRING);
pfile->narrow_cset_desc = save_narrow_cset_desc;
return retval;
/* Subroutine of cpp_interpret_charconst which performs the conversion
to a number, for wide strings. STR is the string structure returned
by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
- cpp_interpret_charconst. */
+ cpp_interpret_charconst. TYPE is the token type. */
static cppchar_t
wide_str_to_charconst (cpp_reader *pfile, cpp_string str,
- unsigned int *pchars_seen, int *unsignedp)
+ unsigned int *pchars_seen, int *unsignedp,
+ enum cpp_ttype type)
{
bool bigend = CPP_OPTION (pfile, bytes_big_endian);
- size_t width = CPP_OPTION (pfile, wchar_precision);
+ size_t width = converter_for_type (pfile, type).width;
size_t cwidth = CPP_OPTION (pfile, char_precision);
size_t mask = width_to_mask (width);
size_t cmask = width_to_mask (cwidth);
/* Wide character constants have type wchar_t, and a single
character exactly fills a wchar_t, so a multi-character wide
character constant is guaranteed to overflow. */
- if (off > 0)
+ if (str.len > nbwc * 2)
cpp_error (pfile, CPP_DL_WARNING,
"character constant too long for its type");
sign- or zero-extend to the full width of cppchar_t. */
if (width < BITS_PER_CPPCHAR_T)
{
- if (CPP_OPTION (pfile, unsigned_wchar) || !(result & (1 << (width - 1))))
+ if (type == CPP_CHAR16 || type == CPP_CHAR32
+ || CPP_OPTION (pfile, unsigned_wchar)
+ || !(result & (1 << (width - 1))))
result &= mask;
else
result |= ~mask;
}
- *unsignedp = CPP_OPTION (pfile, unsigned_wchar);
+ if (type == CPP_CHAR16 || type == CPP_CHAR32
+ || CPP_OPTION (pfile, unsigned_wchar))
+ *unsignedp = 1;
+ else
+ *unsignedp = 0;
+
*pchars_seen = 1;
return result;
}
unsigned int *pchars_seen, int *unsignedp)
{
cpp_string str = { 0, 0 };
- bool wide = (token->type == CPP_WCHAR);
+ bool wide = (token->type != CPP_CHAR);
cppchar_t result;
- /* an empty constant will appear as L'' or '' */
+ /* an empty constant will appear as L'', u'', U'' or '' */
if (token->val.str.len == (size_t) (2 + wide))
{
cpp_error (pfile, CPP_DL_ERROR, "empty character constant");
return 0;
}
- else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, wide))
+ else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, token->type))
return 0;
if (wide)
- result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp);
+ result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp,
+ token->type);
else
result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp);
source file) from INPUT_CHARSET to the source character set. INPUT
points to the input buffer, SIZE is its allocated size, and LEN is
the length of the meaningful data within the buffer. The
- translated buffer is returned, and *ST_SIZE is set to the length of
- the meaningful data within the translated buffer.
-
- INPUT is expected to have been allocated with xmalloc. This function
- will either return INPUT, or free it and return a pointer to another
- xmalloc-allocated block of memory. */
+ translated buffer is returned, *ST_SIZE is set to the length of
+ the meaningful data within the translated buffer, and *BUFFER_START
+ is set to the start of the returned buffer. *BUFFER_START may
+ differ from the return value in the case of a BOM or other ignored
+ marker information.
+
+ INPUT is expected to have been allocated with xmalloc. This
+ function will either set *BUFFER_START to INPUT, or free it and set
+ *BUFFER_START to a pointer to another xmalloc-allocated block of
+ memory. */
uchar *
_cpp_convert_input (cpp_reader *pfile, const char *input_charset,
- uchar *input, size_t size, size_t len, off_t *st_size)
+ uchar *input, size_t size, size_t len,
+ const unsigned char **buffer_start, off_t *st_size)
{
struct cset_converter input_cset;
struct _cpp_strbuf to;
+ unsigned char *buffer;
input_cset = init_iconv_desc (pfile, SOURCE_CHARSET, input_charset);
if (input_cset.func == convert_no_conversion)
else
to.text[to.len] = '\n';
+ buffer = to.text;
*st_size = to.len;
- return to.text;
+#if HOST_CHARSET == HOST_CHARSET_ASCII
+ /* The HOST_CHARSET test just above ensures that the source charset
+ is UTF-8. So, ignore a UTF-8 BOM if we see one. Note that
+ glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
+ BOM -- however, even if it did, we would still need this code due
+ to the 'convert_no_conversion' case. */
+ if (to.len >= 3 && to.text[0] == 0xef && to.text[1] == 0xbb
+ && to.text[2] == 0xbf)
+ {
+ *st_size -= 3;
+ buffer += 3;
+ }
+#endif
+
+ *buffer_start = to.text;
+ return buffer;
}
/* Decide on the default encoding to assume for input files. */