1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2007, 2008, 2009, 2010,
3 2011 Free Software Foundation, Inc.
4 Contributed by Per Bothner, 1994-95.
5 Based on CCCP program by Paul Rubin, June 1986
6 Adapted to ANSI C, Richard Stallman, Jan 1987
7 Broken out to separate file, Zack Weinberg, Mar 2000
9 This program is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 3, or (at your option) any
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
38 enum spell_type category;
39 const unsigned char *name;
42 static const unsigned char *const digraph_spellings[] =
43 { UC"%:", UC"%:%:", UC"<:", UC":>", UC"<%", UC"%>" };
45 #define OP(e, s) { SPELL_OPERATOR, UC s },
46 #define TK(e, s) { SPELL_ ## s, UC #e },
47 static const struct token_spelling token_spellings[N_TTYPES] = { TTYPE_TABLE };
51 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
52 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
54 static void add_line_note (cpp_buffer *, const uchar *, unsigned int);
55 static int skip_line_comment (cpp_reader *);
56 static void skip_whitespace (cpp_reader *, cppchar_t);
57 static void lex_string (cpp_reader *, cpp_token *, const uchar *);
58 static void save_comment (cpp_reader *, cpp_token *, const uchar *, cppchar_t);
59 static void store_comment (cpp_reader *, cpp_token *);
60 static void create_literal (cpp_reader *, cpp_token *, const uchar *,
61 unsigned int, enum cpp_ttype);
62 static bool warn_in_comment (cpp_reader *, _cpp_line_note *);
63 static int name_p (cpp_reader *, const cpp_string *);
64 static tokenrun *next_tokenrun (tokenrun *);
66 static _cpp_buff *new_buff (size_t);
71 Compares, the token TOKEN to the NUL-terminated string STRING.
72 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
74 cpp_ideq (const cpp_token *token, const char *string)
76 if (token->type != CPP_NAME)
79 return !ustrcmp (NODE_NAME (token->val.node.node), (const uchar *) string);
82 /* Record a note TYPE at byte POS into the current cleaned logical
85 add_line_note (cpp_buffer *buffer, const uchar *pos, unsigned int type)
87 if (buffer->notes_used == buffer->notes_cap)
89 buffer->notes_cap = buffer->notes_cap * 2 + 200;
90 buffer->notes = XRESIZEVEC (_cpp_line_note, buffer->notes,
94 buffer->notes[buffer->notes_used].pos = pos;
95 buffer->notes[buffer->notes_used].type = type;
100 /* Fast path to find line special characters using optimized character
101 scanning algorithms. Anything complicated falls back to the slow
102 path below. Since this loop is very hot it's worth doing these kinds
105 One of the paths through the ifdefs should provide
107 const uchar *search_line_fast (const uchar *s, const uchar *end);
109 Between S and END, search for \n, \r, \\, ?. Return a pointer to
112 Note that the last character of the buffer is *always* a newline,
113 as forced by _cpp_convert_input. This fact can be used to avoid
114 explicitly looking for the end of the buffer. */
116 /* Configure gives us an ifdef test. */
117 #ifndef WORDS_BIGENDIAN
118 #define WORDS_BIGENDIAN 0
121 /* We'd like the largest integer that fits into a register. There's nothing
122 in <stdint.h> that gives us that. For most hosts this is unsigned long,
123 but MS decided on an LLP64 model. Thankfully when building with GCC we
124 can get the "real" word size. */
126 typedef unsigned int word_type __attribute__((__mode__(__word__)));
128 typedef unsigned long word_type;
131 /* The code below is only expecting sizes 4 or 8.
132 Die at compile-time if this expectation is violated. */
133 typedef char check_word_type_size
134 [(sizeof(word_type) == 8 || sizeof(word_type) == 4) * 2 - 1];
136 /* Return X with the first N bytes forced to values that won't match one
137 of the interesting characters. Note that NUL is not interesting. */
139 static inline word_type
140 acc_char_mask_misalign (word_type val, unsigned int n)
150 /* Return X replicated to all byte positions within WORD_TYPE. */
152 static inline word_type
153 acc_char_replicate (uchar x)
157 ret = (x << 24) | (x << 16) | (x << 8) | x;
158 if (sizeof(word_type) == 8)
159 ret = (ret << 16 << 16) | ret;
163 /* Return non-zero if some byte of VAL is (probably) C. */
165 static inline word_type
166 acc_char_cmp (word_type val, word_type c)
168 #if defined(__GNUC__) && defined(__alpha__)
169 /* We can get exact results using a compare-bytes instruction.
170 Get (val == c) via (0 >= (val ^ c)). */
171 return __builtin_alpha_cmpbge (0, val ^ c);
173 word_type magic = 0x7efefefeU;
174 if (sizeof(word_type) == 8)
175 magic = (magic << 16 << 16) | 0xfefefefeU;
179 return ((val + magic) ^ ~val) & ~magic;
183 /* Given the result of acc_char_cmp is non-zero, return the index of
184 the found character. If this was a false positive, return -1. */
187 acc_char_index (word_type cmp ATTRIBUTE_UNUSED,
188 word_type val ATTRIBUTE_UNUSED)
190 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN
191 /* The cmpbge instruction sets *bits* of the result corresponding to
192 matches in the bytes with no false positives. */
193 return __builtin_ctzl (cmp);
197 /* ??? It would be nice to force unrolling here,
198 and have all of these constants folded. */
199 for (i = 0; i < sizeof(word_type); ++i)
203 c = (val >> (sizeof(word_type) - i - 1) * 8) & 0xff;
205 c = (val >> i * 8) & 0xff;
207 if (c == '\n' || c == '\r' || c == '\\' || c == '?')
215 /* A version of the fast scanner using bit fiddling techniques.
217 For 32-bit words, one would normally perform 16 comparisons and
218 16 branches. With this algorithm one performs 24 arithmetic
219 operations and one branch. Whether this is faster with a 32-bit
220 word size is going to be somewhat system dependent.
222 For 64-bit words, we eliminate twice the number of comparisons
223 and branches without increasing the number of arithmetic operations.
224 It's almost certainly going to be a win with 64-bit word size. */
226 static const uchar * search_line_acc_char (const uchar *, const uchar *)
230 search_line_acc_char (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
232 const word_type repl_nl = acc_char_replicate ('\n');
233 const word_type repl_cr = acc_char_replicate ('\r');
234 const word_type repl_bs = acc_char_replicate ('\\');
235 const word_type repl_qm = acc_char_replicate ('?');
237 unsigned int misalign;
241 /* Align the buffer. Mask out any bytes from before the beginning. */
242 p = (word_type *)((uintptr_t)s & -sizeof(word_type));
244 misalign = (uintptr_t)s & (sizeof(word_type) - 1);
246 val = acc_char_mask_misalign (val, misalign);
251 t = acc_char_cmp (val, repl_nl);
252 t |= acc_char_cmp (val, repl_cr);
253 t |= acc_char_cmp (val, repl_bs);
254 t |= acc_char_cmp (val, repl_qm);
256 if (__builtin_expect (t != 0, 0))
258 int i = acc_char_index (t, val);
260 return (const uchar *)p + i;
267 /* Disable on Solaris 2/x86 until the following problems can be properly
270 The Solaris 8 assembler cannot assemble SSE2/SSE4.2 insns.
271 The Solaris 9 assembler cannot assemble SSE4.2 insns.
272 Before Solaris 9 Update 6, SSE insns cannot be executed.
273 The Solaris 10+ assembler tags objects with the instruction set
274 extensions used, so SSE4.2 executables cannot run on machines that
275 don't support that extension. */
277 #if (GCC_VERSION >= 4005) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
279 /* Replicated character data to be shared between implementations.
280 Recall that outside of a context with vector support we can't
281 define compatible vector types, therefore these are all defined
282 in terms of raw characters. */
283 static const char repl_chars[4][16] __attribute__((aligned(16))) = {
284 { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
285 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },
286 { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
287 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },
288 { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
289 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },
290 { '?', '?', '?', '?', '?', '?', '?', '?',
291 '?', '?', '?', '?', '?', '?', '?', '?' },
294 /* A version of the fast scanner using MMX vectorized byte compare insns.
296 This uses the PMOVMSKB instruction which was introduced with "MMX2",
297 which was packaged into SSE1; it is also present in the AMD MMX
298 extension. Mark the function as using "sse" so that we emit a real
299 "emms" instruction, rather than the 3dNOW "femms" instruction. */
303 __attribute__((__target__("sse")))
305 search_line_mmx (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
307 typedef char v8qi __attribute__ ((__vector_size__ (8)));
308 typedef int __m64 __attribute__ ((__vector_size__ (8), __may_alias__));
310 const v8qi repl_nl = *(const v8qi *)repl_chars[0];
311 const v8qi repl_cr = *(const v8qi *)repl_chars[1];
312 const v8qi repl_bs = *(const v8qi *)repl_chars[2];
313 const v8qi repl_qm = *(const v8qi *)repl_chars[3];
315 unsigned int misalign, found, mask;
319 /* Align the source pointer. While MMX doesn't generate unaligned data
320 faults, this allows us to safely scan to the end of the buffer without
321 reading beyond the end of the last page. */
322 misalign = (uintptr_t)s & 7;
323 p = (const v8qi *)((uintptr_t)s & -8);
326 /* Create a mask for the bytes that are valid within the first
327 16-byte block. The Idea here is that the AND with the mask
328 within the loop is "free", since we need some AND or TEST
329 insn in order to set the flags for the branch anyway. */
330 mask = -1u << misalign;
332 /* Main loop processing 8 bytes at a time. */
340 t = __builtin_ia32_pcmpeqb(data, repl_nl);
341 c = __builtin_ia32_pcmpeqb(data, repl_cr);
342 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
343 c = __builtin_ia32_pcmpeqb(data, repl_bs);
344 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
345 c = __builtin_ia32_pcmpeqb(data, repl_qm);
346 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
347 found = __builtin_ia32_pmovmskb (t);
352 __builtin_ia32_emms ();
354 /* FOUND contains 1 in bits for which we matched a relevant
355 character. Conversion to the byte index is trivial. */
356 found = __builtin_ctz(found);
357 return (const uchar *)p + found;
360 /* A version of the fast scanner using SSE2 vectorized byte compare insns. */
364 __attribute__((__target__("sse2")))
366 search_line_sse2 (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
368 typedef char v16qi __attribute__ ((__vector_size__ (16)));
370 const v16qi repl_nl = *(const v16qi *)repl_chars[0];
371 const v16qi repl_cr = *(const v16qi *)repl_chars[1];
372 const v16qi repl_bs = *(const v16qi *)repl_chars[2];
373 const v16qi repl_qm = *(const v16qi *)repl_chars[3];
375 unsigned int misalign, found, mask;
379 /* Align the source pointer. */
380 misalign = (uintptr_t)s & 15;
381 p = (const v16qi *)((uintptr_t)s & -16);
384 /* Create a mask for the bytes that are valid within the first
385 16-byte block. The Idea here is that the AND with the mask
386 within the loop is "free", since we need some AND or TEST
387 insn in order to set the flags for the branch anyway. */
388 mask = -1u << misalign;
390 /* Main loop processing 16 bytes at a time. */
398 t = __builtin_ia32_pcmpeqb128(data, repl_nl);
399 t |= __builtin_ia32_pcmpeqb128(data, repl_cr);
400 t |= __builtin_ia32_pcmpeqb128(data, repl_bs);
401 t |= __builtin_ia32_pcmpeqb128(data, repl_qm);
402 found = __builtin_ia32_pmovmskb128 (t);
407 /* FOUND contains 1 in bits for which we matched a relevant
408 character. Conversion to the byte index is trivial. */
409 found = __builtin_ctz(found);
410 return (const uchar *)p + found;
414 /* A version of the fast scanner using SSE 4.2 vectorized string insns. */
418 __attribute__((__target__("sse4.2")))
420 search_line_sse42 (const uchar *s, const uchar *end)
422 typedef char v16qi __attribute__ ((__vector_size__ (16)));
423 static const v16qi search = { '\n', '\r', '?', '\\' };
425 uintptr_t si = (uintptr_t)s;
428 /* Check for unaligned input. */
431 if (__builtin_expect (end - s < 16, 0)
432 && __builtin_expect ((si & 0xfff) > 0xff0, 0))
434 /* There are less than 16 bytes left in the buffer, and less
435 than 16 bytes left on the page. Reading 16 bytes at this
436 point might generate a spurious page fault. Defer to the
437 SSE2 implementation, which already handles alignment. */
438 return search_line_sse2 (s, end);
441 /* ??? The builtin doesn't understand that the PCMPESTRI read from
442 memory need not be aligned. */
443 __asm ("%vpcmpestri $0, (%1), %2"
444 : "=c"(index) : "r"(s), "x"(search), "a"(4), "d"(16));
445 if (__builtin_expect (index < 16, 0))
448 /* Advance the pointer to an aligned address. We will re-scan a
449 few bytes, but we no longer need care for reading past the
450 end of a page, since we're guaranteed a match. */
451 s = (const uchar *)((si + 16) & -16);
454 /* Main loop, processing 16 bytes at a time. By doing the whole loop
455 in inline assembly, we can make proper use of the flags set. */
456 __asm ( "sub $16, %1\n"
459 " %vpcmpestri $0, (%1), %2\n"
461 : "=&c"(index), "+r"(s)
462 : "x"(search), "a"(4), "d"(16));
469 /* Work around out-dated assemblers without sse4 support. */
470 #define search_line_sse42 search_line_sse2
473 /* Check the CPU capabilities. */
475 #include "../gcc/config/i386/cpuid.h"
477 typedef const uchar * (*search_line_fast_type) (const uchar *, const uchar *);
478 static search_line_fast_type search_line_fast;
480 static void __attribute__((constructor))
481 init_vectorized_lexer (void)
483 unsigned dummy, ecx = 0, edx = 0;
484 search_line_fast_type impl = search_line_acc_char;
487 #if defined(__SSE4_2__)
489 #elif defined(__SSE2__)
491 #elif defined(__SSE__)
496 impl = search_line_sse42;
497 else if (__get_cpuid (1, &dummy, &dummy, &ecx, &edx) || minimum == 2)
499 if (minimum == 3 || (ecx & bit_SSE4_2))
500 impl = search_line_sse42;
501 else if (minimum == 2 || (edx & bit_SSE2))
502 impl = search_line_sse2;
503 else if (minimum == 1 || (edx & bit_SSE))
504 impl = search_line_mmx;
506 else if (__get_cpuid (0x80000001, &dummy, &dummy, &dummy, &edx))
509 || (edx & (bit_MMXEXT | bit_CMOV)) == (bit_MMXEXT | bit_CMOV))
510 impl = search_line_mmx;
513 search_line_fast = impl;
516 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__)
518 /* A vection of the fast scanner using AltiVec vectorized byte compares. */
519 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
520 so we can't compile this function without -maltivec on the command line
521 (or implied by some other switch). */
524 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
526 typedef __attribute__((altivec(vector))) unsigned char vc;
529 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
530 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
533 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
534 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
537 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
538 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
541 '?', '?', '?', '?', '?', '?', '?', '?',
542 '?', '?', '?', '?', '?', '?', '?', '?',
545 -1, -1, -1, -1, -1, -1, -1, -1,
546 -1, -1, -1, -1, -1, -1, -1, -1,
548 const vc zero = { 0 };
552 /* Altivec loads automatically mask addresses with -16. This lets us
553 issue the first load as early as possible. */
554 data = __builtin_vec_ld(0, (const vc *)s);
556 /* Discard bytes before the beginning of the buffer. Do this by
557 beginning with all ones and shifting in zeros according to the
558 mis-alignment. The LVSR instruction pulls the exact shift we
559 want from the address. */
560 mask = __builtin_vec_lvsr(0, s);
561 mask = __builtin_vec_perm(zero, ones, mask);
564 /* While altivec loads mask addresses, we still need to align S so
565 that the offset we compute at the end is correct. */
566 s = (const uchar *)((uintptr_t)s & -16);
568 /* Main loop processing 16 bytes at a time. */
572 vc m_nl, m_cr, m_bs, m_qm;
575 data = __builtin_vec_ld(0, (const vc *)s);
578 m_nl = (vc) __builtin_vec_cmpeq(data, repl_nl);
579 m_cr = (vc) __builtin_vec_cmpeq(data, repl_cr);
580 m_bs = (vc) __builtin_vec_cmpeq(data, repl_bs);
581 m_qm = (vc) __builtin_vec_cmpeq(data, repl_qm);
582 t = (m_nl | m_cr) | (m_bs | m_qm);
584 /* T now contains 0xff in bytes for which we matched one of the relevant
585 characters. We want to exit the loop if any byte in T is non-zero.
586 Below is the expansion of vec_any_ne(t, zero). */
588 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t, zero));
591 #define N (sizeof(vc) / sizeof(long))
593 typedef char check_count[(N == 2 || N == 4) * 2 - 1];
598 unsigned long l, i = 0;
602 /* Find the first word of T that is non-zero. */
609 s += sizeof(unsigned long);
613 s += sizeof(unsigned long);
618 s += sizeof(unsigned long);
622 /* L now contains 0xff in bytes for which we matched one of the
623 relevant characters. We can find the byte index by finding
624 its bit index and dividing by 8. */
625 l = __builtin_clzl(l) >> 3;
634 /* We only have one accellerated alternative. Use a direct call so that
635 we encourage inlining. */
637 #define search_line_fast search_line_acc_char
641 /* Returns with a logical line that contains no escaped newlines or
642 trigraphs. This is a time-critical inner loop. */
644 _cpp_clean_line (cpp_reader *pfile)
650 buffer = pfile->buffer;
651 buffer->cur_note = buffer->notes_used = 0;
652 buffer->cur = buffer->line_base = buffer->next_line;
653 buffer->need_line = false;
654 s = buffer->next_line;
656 if (!buffer->from_stage3)
658 const uchar *pbackslash = NULL;
660 /* Fast path. This is the common case of an un-escaped line with
661 no trigraphs. The primary win here is by not writing any
662 data back to memory until we have to. */
665 /* Perform an optimized search for \n, \r, \\, ?. */
666 s = search_line_fast (s, buffer->rlimit);
671 /* Record the location of the backslash and continue. */
674 else if (__builtin_expect (c == '?', 0))
676 if (__builtin_expect (s[1] == '?', false)
677 && _cpp_trigraph_map[s[2]])
679 /* Have a trigraph. We may or may not have to convert
680 it. Add a line note regardless, for -Wtrigraphs. */
681 add_line_note (buffer, s, s[2]);
682 if (CPP_OPTION (pfile, trigraphs))
684 /* We do, and that means we have to switch to the
687 *d = _cpp_trigraph_map[s[2]];
692 /* Not a trigraph. Continue on fast-path. */
699 /* This must be \r or \n. We're either done, or we'll be forced
700 to write back to the buffer and continue on the slow path. */
703 if (__builtin_expect (s == buffer->rlimit, false))
706 /* DOS line ending? */
707 if (__builtin_expect (c == '\r', false) && s[1] == '\n')
710 if (s == buffer->rlimit)
714 if (__builtin_expect (pbackslash == NULL, true))
717 /* Check for escaped newline. */
719 while (is_nvspace (p[-1]))
721 if (p - 1 != pbackslash)
724 /* Have an escaped newline; process it and proceed to
726 add_line_note (buffer, p - 1, p != d ? ' ' : '\\');
728 buffer->next_line = p - 1;
736 if (c == '\n' || c == '\r')
738 /* Handle DOS line endings. */
739 if (c == '\r' && s != buffer->rlimit && s[1] == '\n')
741 if (s == buffer->rlimit)
746 while (p != buffer->next_line && is_nvspace (p[-1]))
748 if (p == buffer->next_line || p[-1] != '\\')
751 add_line_note (buffer, p - 1, p != d ? ' ': '\\');
753 buffer->next_line = p - 1;
755 else if (c == '?' && s[1] == '?' && _cpp_trigraph_map[s[2]])
757 /* Add a note regardless, for the benefit of -Wtrigraphs. */
758 add_line_note (buffer, d, s[2]);
759 if (CPP_OPTION (pfile, trigraphs))
761 *d = _cpp_trigraph_map[s[2]];
769 while (*s != '\n' && *s != '\r')
773 /* Handle DOS line endings. */
774 if (*s == '\r' && s != buffer->rlimit && s[1] == '\n')
780 /* A sentinel note that should never be processed. */
781 add_line_note (buffer, d + 1, '\n');
782 buffer->next_line = s + 1;
785 /* Return true if the trigraph indicated by NOTE should be warned
786 about in a comment. */
788 warn_in_comment (cpp_reader *pfile, _cpp_line_note *note)
792 /* Within comments we don't warn about trigraphs, unless the
793 trigraph forms an escaped newline, as that may change
795 if (note->type != '/')
798 /* If -trigraphs, then this was an escaped newline iff the next note
800 if (CPP_OPTION (pfile, trigraphs))
801 return note[1].pos == note->pos;
803 /* Otherwise, see if this forms an escaped newline. */
805 while (is_nvspace (*p))
808 /* There might have been escaped newlines between the trigraph and the
809 newline we found. Hence the position test. */
810 return (*p == '\n' && p < note[1].pos);
813 /* Process the notes created by add_line_note as far as the current
816 _cpp_process_line_notes (cpp_reader *pfile, int in_comment)
818 cpp_buffer *buffer = pfile->buffer;
822 _cpp_line_note *note = &buffer->notes[buffer->cur_note];
825 if (note->pos > buffer->cur)
829 col = CPP_BUF_COLUMN (buffer, note->pos + 1);
831 if (note->type == '\\' || note->type == ' ')
833 if (note->type == ' ' && !in_comment)
834 cpp_error_with_line (pfile, CPP_DL_WARNING, pfile->line_table->highest_line, col,
835 "backslash and newline separated by space");
837 if (buffer->next_line > buffer->rlimit)
839 cpp_error_with_line (pfile, CPP_DL_PEDWARN, pfile->line_table->highest_line, col,
840 "backslash-newline at end of file");
841 /* Prevent "no newline at end of file" warning. */
842 buffer->next_line = buffer->rlimit;
845 buffer->line_base = note->pos;
846 CPP_INCREMENT_LINE (pfile, 0);
848 else if (_cpp_trigraph_map[note->type])
850 if (CPP_OPTION (pfile, warn_trigraphs)
851 && (!in_comment || warn_in_comment (pfile, note)))
853 if (CPP_OPTION (pfile, trigraphs))
854 cpp_warning_with_line (pfile, CPP_W_TRIGRAPHS,
855 pfile->line_table->highest_line, col,
856 "trigraph ??%c converted to %c",
858 (int) _cpp_trigraph_map[note->type]);
861 cpp_warning_with_line
862 (pfile, CPP_W_TRIGRAPHS,
863 pfile->line_table->highest_line, col,
864 "trigraph ??%c ignored, use -trigraphs to enable",
869 else if (note->type == 0)
870 /* Already processed in lex_raw_string. */;
876 /* Skip a C-style block comment. We find the end of the comment by
877 seeing if an asterisk is before every '/' we encounter. Returns
878 nonzero if comment terminated by EOF, zero otherwise.
880 Buffer->cur points to the initial asterisk of the comment. */
882 _cpp_skip_block_comment (cpp_reader *pfile)
884 cpp_buffer *buffer = pfile->buffer;
885 const uchar *cur = buffer->cur;
894 /* People like decorating comments with '*', so check for '/'
895 instead for efficiency. */
903 /* Warn about potential nested comments, but not if the '/'
904 comes immediately before the true comment delimiter.
905 Don't bother to get it right across escaped newlines. */
906 if (CPP_OPTION (pfile, warn_comments)
907 && cur[0] == '*' && cur[1] != '/')
910 cpp_warning_with_line (pfile, CPP_W_COMMENTS,
911 pfile->line_table->highest_line,
912 CPP_BUF_COL (buffer),
913 "\"/*\" within comment");
919 buffer->cur = cur - 1;
920 _cpp_process_line_notes (pfile, true);
921 if (buffer->next_line >= buffer->rlimit)
923 _cpp_clean_line (pfile);
925 cols = buffer->next_line - buffer->line_base;
926 CPP_INCREMENT_LINE (pfile, cols);
933 _cpp_process_line_notes (pfile, true);
937 /* Skip a C++ line comment, leaving buffer->cur pointing to the
938 terminating newline. Handles escaped newlines. Returns nonzero
939 if a multiline comment. */
941 skip_line_comment (cpp_reader *pfile)
943 cpp_buffer *buffer = pfile->buffer;
944 source_location orig_line = pfile->line_table->highest_line;
946 while (*buffer->cur != '\n')
949 _cpp_process_line_notes (pfile, true);
950 return orig_line != pfile->line_table->highest_line;
953 /* Skips whitespace, saving the next non-whitespace character. */
955 skip_whitespace (cpp_reader *pfile, cppchar_t c)
957 cpp_buffer *buffer = pfile->buffer;
958 bool saw_NUL = false;
962 /* Horizontal space always OK. */
963 if (c == ' ' || c == '\t')
965 /* Just \f \v or \0 left. */
968 else if (pfile->state.in_directive && CPP_PEDANTIC (pfile))
969 cpp_error_with_line (pfile, CPP_DL_PEDWARN, pfile->line_table->highest_line,
970 CPP_BUF_COL (buffer),
971 "%s in preprocessing directive",
972 c == '\f' ? "form feed" : "vertical tab");
976 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
977 while (is_nvspace (c));
980 cpp_error (pfile, CPP_DL_WARNING, "null character(s) ignored");
985 /* See if the characters of a number token are valid in a name (no
988 name_p (cpp_reader *pfile, const cpp_string *string)
992 for (i = 0; i < string->len; i++)
993 if (!is_idchar (string->text[i]))
999 /* After parsing an identifier or other sequence, produce a warning about
1000 sequences not in NFC/NFKC. */
1002 warn_about_normalization (cpp_reader *pfile,
1003 const cpp_token *token,
1004 const struct normalize_state *s)
1006 if (CPP_OPTION (pfile, warn_normalize) < NORMALIZE_STATE_RESULT (s)
1007 && !pfile->state.skipping)
1009 /* Make sure that the token is printed using UCNs, even
1010 if we'd otherwise happily print UTF-8. */
1011 unsigned char *buf = XNEWVEC (unsigned char, cpp_token_len (token));
1014 sz = cpp_spell_token (pfile, token, buf, false) - buf;
1015 if (NORMALIZE_STATE_RESULT (s) == normalized_C)
1016 cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
1017 "`%.*s' is not in NFKC", (int) sz, buf);
1019 cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
1020 "`%.*s' is not in NFC", (int) sz, buf);
1024 /* Returns TRUE if the sequence starting at buffer->cur is invalid in
1025 an identifier. FIRST is TRUE if this starts an identifier. */
1027 forms_identifier_p (cpp_reader *pfile, int first,
1028 struct normalize_state *state)
1030 cpp_buffer *buffer = pfile->buffer;
1032 if (*buffer->cur == '$')
1034 if (!CPP_OPTION (pfile, dollars_in_ident))
1038 if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping)
1040 CPP_OPTION (pfile, warn_dollars) = 0;
1041 cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number");
1047 /* Is this a syntactically valid UCN? */
1048 if (CPP_OPTION (pfile, extended_identifiers)
1049 && *buffer->cur == '\\'
1050 && (buffer->cur[1] == 'u' || buffer->cur[1] == 'U'))
1053 if (_cpp_valid_ucn (pfile, &buffer->cur, buffer->rlimit, 1 + !first,
1062 /* Helper function to get the cpp_hashnode of the identifier BASE. */
1063 static cpp_hashnode *
1064 lex_identifier_intern (cpp_reader *pfile, const uchar *base)
1066 cpp_hashnode *result;
1069 unsigned int hash = HT_HASHSTEP (0, *base);
1072 while (ISIDNUM (*cur))
1074 hash = HT_HASHSTEP (hash, *cur);
1078 hash = HT_HASHFINISH (hash, len);
1079 result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
1080 base, len, hash, HT_ALLOC));
1082 /* Rarely, identifiers require diagnostics when lexed. */
1083 if (__builtin_expect ((result->flags & NODE_DIAGNOSTIC)
1084 && !pfile->state.skipping, 0))
1086 /* It is allowed to poison the same identifier twice. */
1087 if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok)
1088 cpp_error (pfile, CPP_DL_ERROR, "attempt to use poisoned \"%s\"",
1089 NODE_NAME (result));
1091 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1092 replacement list of a variadic macro. */
1093 if (result == pfile->spec_nodes.n__VA_ARGS__
1094 && !pfile->state.va_args_ok)
1095 cpp_error (pfile, CPP_DL_PEDWARN,
1096 "__VA_ARGS__ can only appear in the expansion"
1097 " of a C99 variadic macro");
1099 /* For -Wc++-compat, warn about use of C++ named operators. */
1100 if (result->flags & NODE_WARN_OPERATOR)
1101 cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES,
1102 "identifier \"%s\" is a special operator name in C++",
1103 NODE_NAME (result));
1109 /* Get the cpp_hashnode of an identifier specified by NAME in
1110 the current cpp_reader object. If none is found, NULL is returned. */
1112 _cpp_lex_identifier (cpp_reader *pfile, const char *name)
1114 cpp_hashnode *result;
1115 result = lex_identifier_intern (pfile, (uchar *) name);
1119 /* Lex an identifier starting at BUFFER->CUR - 1. */
1120 static cpp_hashnode *
1121 lex_identifier (cpp_reader *pfile, const uchar *base, bool starts_ucn,
1122 struct normalize_state *nst)
1124 cpp_hashnode *result;
1127 unsigned int hash = HT_HASHSTEP (0, *base);
1129 cur = pfile->buffer->cur;
1131 while (ISIDNUM (*cur))
1133 hash = HT_HASHSTEP (hash, *cur);
1136 pfile->buffer->cur = cur;
1137 if (starts_ucn || forms_identifier_p (pfile, false, nst))
1139 /* Slower version for identifiers containing UCNs (or $). */
1141 while (ISIDNUM (*pfile->buffer->cur))
1143 pfile->buffer->cur++;
1144 NORMALIZE_STATE_UPDATE_IDNUM (nst);
1146 } while (forms_identifier_p (pfile, false, nst));
1147 result = _cpp_interpret_identifier (pfile, base,
1148 pfile->buffer->cur - base);
1153 hash = HT_HASHFINISH (hash, len);
1155 result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
1156 base, len, hash, HT_ALLOC));
1159 /* Rarely, identifiers require diagnostics when lexed. */
1160 if (__builtin_expect ((result->flags & NODE_DIAGNOSTIC)
1161 && !pfile->state.skipping, 0))
1163 /* It is allowed to poison the same identifier twice. */
1164 if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok)
1165 cpp_error (pfile, CPP_DL_ERROR, "attempt to use poisoned \"%s\"",
1166 NODE_NAME (result));
1168 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1169 replacement list of a variadic macro. */
1170 if (result == pfile->spec_nodes.n__VA_ARGS__
1171 && !pfile->state.va_args_ok)
1172 cpp_error (pfile, CPP_DL_PEDWARN,
1173 "__VA_ARGS__ can only appear in the expansion"
1174 " of a C99 variadic macro");
1176 /* For -Wc++-compat, warn about use of C++ named operators. */
1177 if (result->flags & NODE_WARN_OPERATOR)
1178 cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES,
1179 "identifier \"%s\" is a special operator name in C++",
1180 NODE_NAME (result));
1186 /* Lex a number to NUMBER starting at BUFFER->CUR - 1. */
1188 lex_number (cpp_reader *pfile, cpp_string *number,
1189 struct normalize_state *nst)
1195 base = pfile->buffer->cur - 1;
1198 cur = pfile->buffer->cur;
1200 /* N.B. ISIDNUM does not include $. */
1201 while (ISIDNUM (*cur) || *cur == '.' || VALID_SIGN (*cur, cur[-1]))
1204 NORMALIZE_STATE_UPDATE_IDNUM (nst);
1207 pfile->buffer->cur = cur;
1209 while (forms_identifier_p (pfile, false, nst));
1211 number->len = cur - base;
1212 dest = _cpp_unaligned_alloc (pfile, number->len + 1);
1213 memcpy (dest, base, number->len);
1214 dest[number->len] = '\0';
1215 number->text = dest;
1218 /* Create a token of type TYPE with a literal spelling. */
1220 create_literal (cpp_reader *pfile, cpp_token *token, const uchar *base,
1221 unsigned int len, enum cpp_ttype type)
1223 uchar *dest = _cpp_unaligned_alloc (pfile, len + 1);
1225 memcpy (dest, base, len);
1228 token->val.str.len = len;
1229 token->val.str.text = dest;
1232 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
1233 sequence from *FIRST_BUFF_P to LAST_BUFF_P. */
1236 bufring_append (cpp_reader *pfile, const uchar *base, size_t len,
1237 _cpp_buff **first_buff_p, _cpp_buff **last_buff_p)
1239 _cpp_buff *first_buff = *first_buff_p;
1240 _cpp_buff *last_buff = *last_buff_p;
1242 if (first_buff == NULL)
1243 first_buff = last_buff = _cpp_get_buff (pfile, len);
1244 else if (len > BUFF_ROOM (last_buff))
1246 size_t room = BUFF_ROOM (last_buff);
1247 memcpy (BUFF_FRONT (last_buff), base, room);
1248 BUFF_FRONT (last_buff) += room;
1251 last_buff = _cpp_append_extend_buff (pfile, last_buff, len);
1254 memcpy (BUFF_FRONT (last_buff), base, len);
1255 BUFF_FRONT (last_buff) += len;
1257 *first_buff_p = first_buff;
1258 *last_buff_p = last_buff;
1261 /* Lexes a raw string. The stored string contains the spelling, including
1262 double quotes, delimiter string, '(' and ')', any leading
1263 'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the
1264 literal, or CPP_OTHER if it was not properly terminated.
1266 The spelling is NUL-terminated, but it is not guaranteed that this
1267 is the first NUL since embedded NULs are preserved. */
1270 lex_raw_string (cpp_reader *pfile, cpp_token *token, const uchar *base,
1273 const uchar *raw_prefix;
1274 unsigned int raw_prefix_len = 0;
1275 enum cpp_ttype type;
1276 size_t total_len = 0;
1277 _cpp_buff *first_buff = NULL, *last_buff = NULL;
1278 _cpp_line_note *note = &pfile->buffer->notes[pfile->buffer->cur_note];
1280 type = (*base == 'L' ? CPP_WSTRING :
1281 *base == 'U' ? CPP_STRING32 :
1282 *base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16)
1285 raw_prefix = cur + 1;
1286 while (raw_prefix_len < 16)
1288 switch (raw_prefix[raw_prefix_len])
1290 case ' ': case '(': case ')': case '\\': case '\t':
1291 case '\v': case '\f': case '\n': default:
1293 /* Basic source charset except the above chars. */
1294 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1295 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1296 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1297 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1299 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1300 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1301 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1302 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1304 case '0': case '1': case '2': case '3': case '4': case '5':
1305 case '6': case '7': case '8': case '9':
1306 case '_': case '{': case '}': case '#': case '[': case ']':
1307 case '<': case '>': case '%': case ':': case ';': case '.':
1308 case '?': case '*': case '+': case '-': case '/': case '^':
1309 case '&': case '|': case '~': case '!': case '=': case ',':
1310 case '"': case '\'':
1317 if (raw_prefix[raw_prefix_len] != '(')
1319 int col = CPP_BUF_COLUMN (pfile->buffer, raw_prefix + raw_prefix_len)
1321 if (raw_prefix_len == 16)
1322 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, col,
1323 "raw string delimiter longer than 16 characters");
1325 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, col,
1326 "invalid character '%c' in raw string delimiter",
1327 (int) raw_prefix[raw_prefix_len]);
1328 pfile->buffer->cur = raw_prefix - 1;
1329 create_literal (pfile, token, base, raw_prefix - 1 - base, CPP_OTHER);
1333 cur = raw_prefix + raw_prefix_len + 1;
1336 #define BUF_APPEND(STR,LEN) \
1338 bufring_append (pfile, (const uchar *)(STR), (LEN), \
1339 &first_buff, &last_buff); \
1340 total_len += (LEN); \
1345 /* If we previously performed any trigraph or line splicing
1346 transformations, undo them within the body of the raw string. */
1347 while (note->pos < cur)
1349 for (; note->pos == cur; ++note)
1355 /* Restore backslash followed by newline. */
1356 BUF_APPEND (base, cur - base);
1358 BUF_APPEND ("\\", 1);
1360 if (note->type == ' ')
1362 /* GNU backslash whitespace newline extension. FIXME
1363 could be any sequence of non-vertical space. When we
1364 can properly restore any such sequence, we should mark
1365 this note as handled so _cpp_process_line_notes
1367 BUF_APPEND (" ", 1);
1370 BUF_APPEND ("\n", 1);
1374 /* Already handled. */
1378 if (_cpp_trigraph_map[note->type])
1380 /* Don't warn about this trigraph in
1381 _cpp_process_line_notes, since trigraphs show up as
1382 trigraphs in raw strings. */
1383 uchar type = note->type;
1386 if (!CPP_OPTION (pfile, trigraphs))
1387 /* If we didn't convert the trigraph in the first
1388 place, don't do anything now either. */
1391 BUF_APPEND (base, cur - base);
1393 BUF_APPEND ("??", 2);
1395 /* ??/ followed by newline gets two line notes, one for
1396 the trigraph and one for the backslash/newline. */
1397 if (type == '/' && note[1].pos == cur)
1399 if (note[1].type != '\\'
1400 && note[1].type != ' ')
1402 BUF_APPEND ("/", 1);
1404 goto after_backslash;
1406 /* The ) from ??) could be part of the suffix. */
1407 else if (type == ')'
1408 && strncmp ((const char *) cur+1,
1409 (const char *) raw_prefix,
1410 raw_prefix_len) == 0
1411 && cur[raw_prefix_len+1] == '"')
1413 BUF_APPEND (")", 1);
1415 cur += raw_prefix_len + 2;
1416 goto break_outer_loop;
1420 /* Skip the replacement character. */
1422 BUF_APPEND (&type, 1);
1433 && strncmp ((const char *) cur, (const char *) raw_prefix,
1434 raw_prefix_len) == 0
1435 && cur[raw_prefix_len] == '"')
1437 cur += raw_prefix_len + 1;
1442 if (pfile->state.in_directive
1443 || pfile->state.parsing_args
1444 || pfile->state.in_deferred_pragma)
1448 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 0,
1449 "unterminated raw string");
1453 BUF_APPEND (base, cur - base);
1455 if (pfile->buffer->cur < pfile->buffer->rlimit)
1456 CPP_INCREMENT_LINE (pfile, 0);
1457 pfile->buffer->need_line = true;
1459 pfile->buffer->cur = cur-1;
1460 _cpp_process_line_notes (pfile, false);
1461 if (!_cpp_get_fresh_line (pfile))
1463 source_location src_loc = token->src_loc;
1464 token->type = CPP_EOF;
1465 /* Tell the compiler the line number of the EOF token. */
1466 token->src_loc = pfile->line_table->highest_line;
1468 if (first_buff != NULL)
1469 _cpp_release_buff (pfile, first_buff);
1470 cpp_error_with_line (pfile, CPP_DL_ERROR, src_loc, 0,
1471 "unterminated raw string");
1475 cur = base = pfile->buffer->cur;
1476 note = &pfile->buffer->notes[pfile->buffer->cur_note];
1481 pfile->buffer->cur = cur;
1482 if (first_buff == NULL)
1483 create_literal (pfile, token, base, cur - base, type);
1486 uchar *dest = _cpp_unaligned_alloc (pfile, total_len + (cur - base) + 1);
1489 token->val.str.len = total_len + (cur - base);
1490 token->val.str.text = dest;
1491 last_buff = first_buff;
1492 while (last_buff != NULL)
1494 memcpy (dest, last_buff->base,
1495 BUFF_FRONT (last_buff) - last_buff->base);
1496 dest += BUFF_FRONT (last_buff) - last_buff->base;
1497 last_buff = last_buff->next;
1499 _cpp_release_buff (pfile, first_buff);
1500 memcpy (dest, base, cur - base);
1501 dest[cur - base] = '\0';
1505 /* Lexes a string, character constant, or angle-bracketed header file
1506 name. The stored string contains the spelling, including opening
1507 quote and any leading 'L', 'u', 'U' or 'u8' and optional
1508 'R' modifier. It returns the type of the literal, or CPP_OTHER
1509 if it was not properly terminated, or CPP_LESS for an unterminated
1510 header name which must be relexed as normal tokens.
1512 The spelling is NUL-terminated, but it is not guaranteed that this
1513 is the first NUL since embedded NULs are preserved. */
1515 lex_string (cpp_reader *pfile, cpp_token *token, const uchar *base)
1517 bool saw_NUL = false;
1519 cppchar_t terminator;
1520 enum cpp_ttype type;
1523 terminator = *cur++;
1524 if (terminator == 'L' || terminator == 'U')
1525 terminator = *cur++;
1526 else if (terminator == 'u')
1528 terminator = *cur++;
1529 if (terminator == '8')
1530 terminator = *cur++;
1532 if (terminator == 'R')
1534 lex_raw_string (pfile, token, base, cur);
1537 if (terminator == '"')
1538 type = (*base == 'L' ? CPP_WSTRING :
1539 *base == 'U' ? CPP_STRING32 :
1540 *base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16)
1542 else if (terminator == '\'')
1543 type = (*base == 'L' ? CPP_WCHAR :
1544 *base == 'U' ? CPP_CHAR32 :
1545 *base == 'u' ? CPP_CHAR16 : CPP_CHAR);
1547 terminator = '>', type = CPP_HEADER_NAME;
1551 cppchar_t c = *cur++;
1553 /* In #include-style directives, terminators are not escapable. */
1554 if (c == '\\' && !pfile->state.angled_headers && *cur != '\n')
1556 else if (c == terminator)
1561 /* Unmatched quotes always yield undefined behavior, but
1562 greedy lexing means that what appears to be an unterminated
1563 header name may actually be a legitimate sequence of tokens. */
1564 if (terminator == '>')
1566 token->type = CPP_LESS;
1576 if (saw_NUL && !pfile->state.skipping)
1577 cpp_error (pfile, CPP_DL_WARNING,
1578 "null character(s) preserved in literal");
1580 if (type == CPP_OTHER && CPP_OPTION (pfile, lang) != CLK_ASM)
1581 cpp_error (pfile, CPP_DL_PEDWARN, "missing terminating %c character",
1584 pfile->buffer->cur = cur;
1585 create_literal (pfile, token, base, cur - base, type);
1588 /* Return the comment table. The client may not make any assumption
1589 about the ordering of the table. */
1591 cpp_get_comments (cpp_reader *pfile)
1593 return &pfile->comments;
1596 /* Append a comment to the end of the comment table. */
1598 store_comment (cpp_reader *pfile, cpp_token *token)
1602 if (pfile->comments.allocated == 0)
1604 pfile->comments.allocated = 256;
1605 pfile->comments.entries = (cpp_comment *) xmalloc
1606 (pfile->comments.allocated * sizeof (cpp_comment));
1609 if (pfile->comments.count == pfile->comments.allocated)
1611 pfile->comments.allocated *= 2;
1612 pfile->comments.entries = (cpp_comment *) xrealloc
1613 (pfile->comments.entries,
1614 pfile->comments.allocated * sizeof (cpp_comment));
1617 len = token->val.str.len;
1619 /* Copy comment. Note, token may not be NULL terminated. */
1620 pfile->comments.entries[pfile->comments.count].comment =
1621 (char *) xmalloc (sizeof (char) * (len + 1));
1622 memcpy (pfile->comments.entries[pfile->comments.count].comment,
1623 token->val.str.text, len);
1624 pfile->comments.entries[pfile->comments.count].comment[len] = '\0';
1626 /* Set source location. */
1627 pfile->comments.entries[pfile->comments.count].sloc = token->src_loc;
1629 /* Increment the count of entries in the comment table. */
1630 pfile->comments.count++;
1633 /* The stored comment includes the comment start and any terminator. */
1635 save_comment (cpp_reader *pfile, cpp_token *token, const unsigned char *from,
1638 unsigned char *buffer;
1639 unsigned int len, clen, i;
1641 len = pfile->buffer->cur - from + 1; /* + 1 for the initial '/'. */
1643 /* C++ comments probably (not definitely) have moved past a new
1644 line, which we don't want to save in the comment. */
1645 if (is_vspace (pfile->buffer->cur[-1]))
1648 /* If we are currently in a directive or in argument parsing, then
1649 we need to store all C++ comments as C comments internally, and
1650 so we need to allocate a little extra space in that case.
1652 Note that the only time we encounter a directive here is
1653 when we are saving comments in a "#define". */
1654 clen = ((pfile->state.in_directive || pfile->state.parsing_args)
1655 && type == '/') ? len + 2 : len;
1657 buffer = _cpp_unaligned_alloc (pfile, clen);
1659 token->type = CPP_COMMENT;
1660 token->val.str.len = clen;
1661 token->val.str.text = buffer;
1664 memcpy (buffer + 1, from, len - 1);
1666 /* Finish conversion to a C comment, if necessary. */
1667 if ((pfile->state.in_directive || pfile->state.parsing_args) && type == '/')
1670 buffer[clen - 2] = '*';
1671 buffer[clen - 1] = '/';
1672 /* As there can be in a C++ comments illegal sequences for C comments
1673 we need to filter them out. */
1674 for (i = 2; i < (clen - 2); i++)
1675 if (buffer[i] == '/' && (buffer[i - 1] == '*' || buffer[i + 1] == '*'))
1679 /* Finally store this comment for use by clients of libcpp. */
1680 store_comment (pfile, token);
1683 /* Allocate COUNT tokens for RUN. */
1685 _cpp_init_tokenrun (tokenrun *run, unsigned int count)
1687 run->base = XNEWVEC (cpp_token, count);
1688 run->limit = run->base + count;
1692 /* Returns the next tokenrun, or creates one if there is none. */
1694 next_tokenrun (tokenrun *run)
1696 if (run->next == NULL)
1698 run->next = XNEW (tokenrun);
1699 run->next->prev = run;
1700 _cpp_init_tokenrun (run->next, 250);
1706 /* Return the number of not yet processed token in the the current
1709 _cpp_remaining_tokens_num_in_context (cpp_reader *pfile)
1711 cpp_context *context = pfile->context;
1712 if (context->tokens_kind == TOKENS_KIND_DIRECT)
1713 return ((LAST (context).token - FIRST (context).token)
1714 / sizeof (cpp_token));
1715 else if (context->tokens_kind == TOKENS_KIND_INDIRECT
1716 || context->tokens_kind == TOKENS_KIND_EXTENDED)
1717 return ((LAST (context).ptoken - FIRST (context).ptoken)
1718 / sizeof (cpp_token *));
1723 /* Returns the token present at index INDEX in the current context.
1724 If INDEX is zero, the next token to be processed is returned. */
1725 static const cpp_token*
1726 _cpp_token_from_context_at (cpp_reader *pfile, int index)
1728 cpp_context *context = pfile->context;
1729 if (context->tokens_kind == TOKENS_KIND_DIRECT)
1730 return &(FIRST (context).token[index]);
1731 else if (context->tokens_kind == TOKENS_KIND_INDIRECT
1732 || context->tokens_kind == TOKENS_KIND_EXTENDED)
1733 return FIRST (context).ptoken[index];
1738 /* Look ahead in the input stream. */
1740 cpp_peek_token (cpp_reader *pfile, int index)
1742 cpp_context *context = pfile->context;
1743 const cpp_token *peektok;
1746 /* First, scan through any pending cpp_context objects. */
1747 while (context->prev)
1749 ptrdiff_t sz = _cpp_remaining_tokens_num_in_context (pfile);
1751 if (index < (int) sz)
1752 return _cpp_token_from_context_at (pfile, index);
1754 context = context->prev;
1757 /* We will have to read some new tokens after all (and do so
1758 without invalidating preceding tokens). */
1760 pfile->keep_tokens++;
1764 peektok = _cpp_lex_token (pfile);
1765 if (peektok->type == CPP_EOF)
1770 _cpp_backup_tokens_direct (pfile, count + 1);
1771 pfile->keep_tokens--;
1776 /* Allocate a single token that is invalidated at the same time as the
1777 rest of the tokens on the line. Has its line and col set to the
1778 same as the last lexed token, so that diagnostics appear in the
1781 _cpp_temp_token (cpp_reader *pfile)
1783 cpp_token *old, *result;
1784 ptrdiff_t sz = pfile->cur_run->limit - pfile->cur_token;
1785 ptrdiff_t la = (ptrdiff_t) pfile->lookaheads;
1787 old = pfile->cur_token - 1;
1788 /* Any pre-existing lookaheads must not be clobbered. */
1793 tokenrun *next = next_tokenrun (pfile->cur_run);
1796 memmove (next->base + 1, next->base,
1797 (la - sz) * sizeof (cpp_token));
1799 next->base[0] = pfile->cur_run->limit[-1];
1803 memmove (pfile->cur_token + 1, pfile->cur_token,
1804 MIN (la, sz - 1) * sizeof (cpp_token));
1807 if (!sz && pfile->cur_token == pfile->cur_run->limit)
1809 pfile->cur_run = next_tokenrun (pfile->cur_run);
1810 pfile->cur_token = pfile->cur_run->base;
1813 result = pfile->cur_token++;
1814 result->src_loc = old->src_loc;
1818 /* Lex a token into RESULT (external interface). Takes care of issues
1819 like directive handling, token lookahead, multiple include
1820 optimization and skipping. */
1822 _cpp_lex_token (cpp_reader *pfile)
1828 if (pfile->cur_token == pfile->cur_run->limit)
1830 pfile->cur_run = next_tokenrun (pfile->cur_run);
1831 pfile->cur_token = pfile->cur_run->base;
1833 /* We assume that the current token is somewhere in the current
1835 if (pfile->cur_token < pfile->cur_run->base
1836 || pfile->cur_token >= pfile->cur_run->limit)
1839 if (pfile->lookaheads)
1841 pfile->lookaheads--;
1842 result = pfile->cur_token++;
1845 result = _cpp_lex_direct (pfile);
1847 if (result->flags & BOL)
1849 /* Is this a directive. If _cpp_handle_directive returns
1850 false, it is an assembler #. */
1851 if (result->type == CPP_HASH
1852 /* 6.10.3 p 11: Directives in a list of macro arguments
1853 gives undefined behavior. This implementation
1854 handles the directive as normal. */
1855 && pfile->state.parsing_args != 1)
1857 if (_cpp_handle_directive (pfile, result->flags & PREV_WHITE))
1859 if (pfile->directive_result.type == CPP_PADDING)
1861 result = &pfile->directive_result;
1864 else if (pfile->state.in_deferred_pragma)
1865 result = &pfile->directive_result;
1867 if (pfile->cb.line_change && !pfile->state.skipping)
1868 pfile->cb.line_change (pfile, result, pfile->state.parsing_args);
1871 /* We don't skip tokens in directives. */
1872 if (pfile->state.in_directive || pfile->state.in_deferred_pragma)
1875 /* Outside a directive, invalidate controlling macros. At file
1876 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
1877 get here and MI optimization works. */
1878 pfile->mi_valid = false;
1880 if (!pfile->state.skipping || result->type == CPP_EOF)
1887 /* Returns true if a fresh line has been loaded. */
1889 _cpp_get_fresh_line (cpp_reader *pfile)
1893 /* We can't get a new line until we leave the current directive. */
1894 if (pfile->state.in_directive)
1899 cpp_buffer *buffer = pfile->buffer;
1901 if (!buffer->need_line)
1904 if (buffer->next_line < buffer->rlimit)
1906 _cpp_clean_line (pfile);
1910 /* First, get out of parsing arguments state. */
1911 if (pfile->state.parsing_args)
1914 /* End of buffer. Non-empty files should end in a newline. */
1915 if (buffer->buf != buffer->rlimit
1916 && buffer->next_line > buffer->rlimit
1917 && !buffer->from_stage3)
1919 /* Clip to buffer size. */
1920 buffer->next_line = buffer->rlimit;
1923 return_at_eof = buffer->return_at_eof;
1924 _cpp_pop_buffer (pfile);
1925 if (pfile->buffer == NULL || return_at_eof)
1930 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
1933 result->type = ELSE_TYPE; \
1934 if (*buffer->cur == CHAR) \
1935 buffer->cur++, result->type = THEN_TYPE; \
1939 /* Lex a token into pfile->cur_token, which is also incremented, to
1940 get diagnostics pointing to the correct location.
1942 Does not handle issues such as token lookahead, multiple-include
1943 optimization, directives, skipping etc. This function is only
1944 suitable for use by _cpp_lex_token, and in special cases like
1945 lex_expansion_token which doesn't care for any of these issues.
1947 When meeting a newline, returns CPP_EOF if parsing a directive,
1948 otherwise returns to the start of the token buffer if permissible.
1949 Returns the location of the lexed token. */
1951 _cpp_lex_direct (cpp_reader *pfile)
1955 const unsigned char *comment_start;
1956 cpp_token *result = pfile->cur_token++;
1960 buffer = pfile->buffer;
1961 if (buffer->need_line)
1963 if (pfile->state.in_deferred_pragma)
1965 result->type = CPP_PRAGMA_EOL;
1966 pfile->state.in_deferred_pragma = false;
1967 if (!pfile->state.pragma_allow_expansion)
1968 pfile->state.prevent_expansion--;
1971 if (!_cpp_get_fresh_line (pfile))
1973 result->type = CPP_EOF;
1974 if (!pfile->state.in_directive)
1976 /* Tell the compiler the line number of the EOF token. */
1977 result->src_loc = pfile->line_table->highest_line;
1978 result->flags = BOL;
1982 if (!pfile->keep_tokens)
1984 pfile->cur_run = &pfile->base_run;
1985 result = pfile->base_run.base;
1986 pfile->cur_token = result + 1;
1988 result->flags = BOL;
1989 if (pfile->state.parsing_args == 2)
1990 result->flags |= PREV_WHITE;
1992 buffer = pfile->buffer;
1994 result->src_loc = pfile->line_table->highest_line;
1997 if (buffer->cur >= buffer->notes[buffer->cur_note].pos
1998 && !pfile->overlaid_buffer)
2000 _cpp_process_line_notes (pfile, false);
2001 result->src_loc = pfile->line_table->highest_line;
2005 if (pfile->forced_token_location_p)
2006 result->src_loc = *pfile->forced_token_location_p;
2008 result->src_loc = linemap_position_for_column (pfile->line_table,
2009 CPP_BUF_COLUMN (buffer, buffer->cur));
2013 case ' ': case '\t': case '\f': case '\v': case '\0':
2014 result->flags |= PREV_WHITE;
2015 skip_whitespace (pfile, c);
2019 if (buffer->cur < buffer->rlimit)
2020 CPP_INCREMENT_LINE (pfile, 0);
2021 buffer->need_line = true;
2024 case '0': case '1': case '2': case '3': case '4':
2025 case '5': case '6': case '7': case '8': case '9':
2027 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2028 result->type = CPP_NUMBER;
2029 lex_number (pfile, &result->val.str, &nst);
2030 warn_about_normalization (pfile, result, &nst);
2038 /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters,
2039 wide strings or raw strings. */
2040 if (c == 'L' || CPP_OPTION (pfile, rliterals)
2041 || (c != 'R' && CPP_OPTION (pfile, uliterals)))
2043 if ((*buffer->cur == '\'' && c != 'R')
2044 || *buffer->cur == '"'
2045 || (*buffer->cur == 'R'
2047 && buffer->cur[1] == '"'
2048 && CPP_OPTION (pfile, rliterals))
2049 || (*buffer->cur == '8'
2051 && (buffer->cur[1] == '"'
2052 || (buffer->cur[1] == 'R' && buffer->cur[2] == '"'
2053 && CPP_OPTION (pfile, rliterals)))))
2055 lex_string (pfile, result, buffer->cur - 1);
2062 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2063 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2064 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2065 case 's': case 't': case 'v': case 'w': case 'x':
2067 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2068 case 'G': case 'H': case 'I': case 'J': case 'K':
2069 case 'M': case 'N': case 'O': case 'P': case 'Q':
2070 case 'S': case 'T': case 'V': case 'W': case 'X':
2072 result->type = CPP_NAME;
2074 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2075 result->val.node.node = lex_identifier (pfile, buffer->cur - 1, false,
2077 warn_about_normalization (pfile, result, &nst);
2080 /* Convert named operators to their proper types. */
2081 if (result->val.node.node->flags & NODE_OPERATOR)
2083 result->flags |= NAMED_OP;
2084 result->type = (enum cpp_ttype) result->val.node.node->directive_index;
2090 lex_string (pfile, result, buffer->cur - 1);
2094 /* A potential block or line comment. */
2095 comment_start = buffer->cur;
2100 if (_cpp_skip_block_comment (pfile))
2101 cpp_error (pfile, CPP_DL_ERROR, "unterminated comment");
2103 else if (c == '/' && (CPP_OPTION (pfile, cplusplus_comments)
2104 || cpp_in_system_header (pfile)))
2106 /* Warn about comments only if pedantically GNUC89, and not
2107 in system headers. */
2108 if (CPP_OPTION (pfile, lang) == CLK_GNUC89 && CPP_PEDANTIC (pfile)
2109 && ! buffer->warned_cplusplus_comments)
2111 cpp_error (pfile, CPP_DL_PEDWARN,
2112 "C++ style comments are not allowed in ISO C90");
2113 cpp_error (pfile, CPP_DL_PEDWARN,
2114 "(this will be reported only once per input file)");
2115 buffer->warned_cplusplus_comments = 1;
2118 if (skip_line_comment (pfile) && CPP_OPTION (pfile, warn_comments))
2119 cpp_warning (pfile, CPP_W_COMMENTS, "multi-line comment");
2124 result->type = CPP_DIV_EQ;
2129 result->type = CPP_DIV;
2133 if (!pfile->state.save_comments)
2135 result->flags |= PREV_WHITE;
2136 goto update_tokens_line;
2139 /* Save the comment as a token in its own right. */
2140 save_comment (pfile, result, comment_start, c);
2144 if (pfile->state.angled_headers)
2146 lex_string (pfile, result, buffer->cur - 1);
2147 if (result->type != CPP_LESS)
2151 result->type = CPP_LESS;
2152 if (*buffer->cur == '=')
2153 buffer->cur++, result->type = CPP_LESS_EQ;
2154 else if (*buffer->cur == '<')
2157 IF_NEXT_IS ('=', CPP_LSHIFT_EQ, CPP_LSHIFT);
2159 else if (CPP_OPTION (pfile, digraphs))
2161 if (*buffer->cur == ':')
2164 result->flags |= DIGRAPH;
2165 result->type = CPP_OPEN_SQUARE;
2167 else if (*buffer->cur == '%')
2170 result->flags |= DIGRAPH;
2171 result->type = CPP_OPEN_BRACE;
2177 result->type = CPP_GREATER;
2178 if (*buffer->cur == '=')
2179 buffer->cur++, result->type = CPP_GREATER_EQ;
2180 else if (*buffer->cur == '>')
2183 IF_NEXT_IS ('=', CPP_RSHIFT_EQ, CPP_RSHIFT);
2188 result->type = CPP_MOD;
2189 if (*buffer->cur == '=')
2190 buffer->cur++, result->type = CPP_MOD_EQ;
2191 else if (CPP_OPTION (pfile, digraphs))
2193 if (*buffer->cur == ':')
2196 result->flags |= DIGRAPH;
2197 result->type = CPP_HASH;
2198 if (*buffer->cur == '%' && buffer->cur[1] == ':')
2199 buffer->cur += 2, result->type = CPP_PASTE, result->val.token_no = 0;
2201 else if (*buffer->cur == '>')
2204 result->flags |= DIGRAPH;
2205 result->type = CPP_CLOSE_BRACE;
2211 result->type = CPP_DOT;
2212 if (ISDIGIT (*buffer->cur))
2214 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2215 result->type = CPP_NUMBER;
2216 lex_number (pfile, &result->val.str, &nst);
2217 warn_about_normalization (pfile, result, &nst);
2219 else if (*buffer->cur == '.' && buffer->cur[1] == '.')
2220 buffer->cur += 2, result->type = CPP_ELLIPSIS;
2221 else if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus))
2222 buffer->cur++, result->type = CPP_DOT_STAR;
2226 result->type = CPP_PLUS;
2227 if (*buffer->cur == '+')
2228 buffer->cur++, result->type = CPP_PLUS_PLUS;
2229 else if (*buffer->cur == '=')
2230 buffer->cur++, result->type = CPP_PLUS_EQ;
2234 result->type = CPP_MINUS;
2235 if (*buffer->cur == '>')
2238 result->type = CPP_DEREF;
2239 if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus))
2240 buffer->cur++, result->type = CPP_DEREF_STAR;
2242 else if (*buffer->cur == '-')
2243 buffer->cur++, result->type = CPP_MINUS_MINUS;
2244 else if (*buffer->cur == '=')
2245 buffer->cur++, result->type = CPP_MINUS_EQ;
2249 result->type = CPP_AND;
2250 if (*buffer->cur == '&')
2251 buffer->cur++, result->type = CPP_AND_AND;
2252 else if (*buffer->cur == '=')
2253 buffer->cur++, result->type = CPP_AND_EQ;
2257 result->type = CPP_OR;
2258 if (*buffer->cur == '|')
2259 buffer->cur++, result->type = CPP_OR_OR;
2260 else if (*buffer->cur == '=')
2261 buffer->cur++, result->type = CPP_OR_EQ;
2265 result->type = CPP_COLON;
2266 if (*buffer->cur == ':' && CPP_OPTION (pfile, cplusplus))
2267 buffer->cur++, result->type = CPP_SCOPE;
2268 else if (*buffer->cur == '>' && CPP_OPTION (pfile, digraphs))
2271 result->flags |= DIGRAPH;
2272 result->type = CPP_CLOSE_SQUARE;
2276 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ, CPP_MULT); break;
2277 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ, CPP_EQ); break;
2278 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ, CPP_NOT); break;
2279 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ, CPP_XOR); break;
2280 case '#': IF_NEXT_IS ('#', CPP_PASTE, CPP_HASH); result->val.token_no = 0; break;
2282 case '?': result->type = CPP_QUERY; break;
2283 case '~': result->type = CPP_COMPL; break;
2284 case ',': result->type = CPP_COMMA; break;
2285 case '(': result->type = CPP_OPEN_PAREN; break;
2286 case ')': result->type = CPP_CLOSE_PAREN; break;
2287 case '[': result->type = CPP_OPEN_SQUARE; break;
2288 case ']': result->type = CPP_CLOSE_SQUARE; break;
2289 case '{': result->type = CPP_OPEN_BRACE; break;
2290 case '}': result->type = CPP_CLOSE_BRACE; break;
2291 case ';': result->type = CPP_SEMICOLON; break;
2293 /* @ is a punctuator in Objective-C. */
2294 case '@': result->type = CPP_ATSIGN; break;
2299 const uchar *base = --buffer->cur;
2300 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2302 if (forms_identifier_p (pfile, true, &nst))
2304 result->type = CPP_NAME;
2305 result->val.node.node = lex_identifier (pfile, base, true, &nst);
2306 warn_about_normalization (pfile, result, &nst);
2313 create_literal (pfile, result, buffer->cur - 1, 1, CPP_OTHER);
2320 /* An upper bound on the number of bytes needed to spell TOKEN.
2321 Does not include preceding whitespace. */
2323 cpp_token_len (const cpp_token *token)
2327 switch (TOKEN_SPELL (token))
2329 default: len = 6; break;
2330 case SPELL_LITERAL: len = token->val.str.len; break;
2331 case SPELL_IDENT: len = NODE_LEN (token->val.node.node) * 10; break;
2337 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER.
2338 Return the number of bytes read out of NAME. (There are always
2339 10 bytes written to BUFFER.) */
2342 utf8_to_ucn (unsigned char *buffer, const unsigned char *name)
2348 unsigned long utf32;
2350 /* Compute the length of the UTF-8 sequence. */
2351 for (t = *name; t & 0x80; t <<= 1)
2354 utf32 = *name & (0x7F >> ucn_len);
2355 for (ucn_len_c = 1; ucn_len_c < ucn_len; ucn_len_c++)
2357 utf32 = (utf32 << 6) | (*++name & 0x3F);
2359 /* Ill-formed UTF-8. */
2360 if ((*name & ~0x3F) != 0x80)
2366 for (j = 7; j >= 0; j--)
2367 *buffer++ = "0123456789abcdef"[(utf32 >> (4 * j)) & 0xF];
2371 /* Given a token TYPE corresponding to a digraph, return a pointer to
2372 the spelling of the digraph. */
2373 static const unsigned char *
2374 cpp_digraph2name (enum cpp_ttype type)
2376 return digraph_spellings[(int) type - (int) CPP_FIRST_DIGRAPH];
2379 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
2380 already contain the enough space to hold the token's spelling.
2381 Returns a pointer to the character after the last character written.
2382 FORSTRING is true if this is to be the spelling after translation
2383 phase 1 (this is different for UCNs).
2384 FIXME: Would be nice if we didn't need the PFILE argument. */
2386 cpp_spell_token (cpp_reader *pfile, const cpp_token *token,
2387 unsigned char *buffer, bool forstring)
2389 switch (TOKEN_SPELL (token))
2391 case SPELL_OPERATOR:
2393 const unsigned char *spelling;
2396 if (token->flags & DIGRAPH)
2397 spelling = cpp_digraph2name (token->type);
2398 else if (token->flags & NAMED_OP)
2401 spelling = TOKEN_NAME (token);
2403 while ((c = *spelling++) != '\0')
2412 memcpy (buffer, NODE_NAME (token->val.node.node),
2413 NODE_LEN (token->val.node.node));
2414 buffer += NODE_LEN (token->val.node.node);
2419 const unsigned char * name = NODE_NAME (token->val.node.node);
2421 for (i = 0; i < NODE_LEN (token->val.node.node); i++)
2422 if (name[i] & ~0x7F)
2424 i += utf8_to_ucn (buffer, name + i) - 1;
2428 *buffer++ = NODE_NAME (token->val.node.node)[i];
2433 memcpy (buffer, token->val.str.text, token->val.str.len);
2434 buffer += token->val.str.len;
2438 cpp_error (pfile, CPP_DL_ICE,
2439 "unspellable token %s", TOKEN_NAME (token));
2446 /* Returns TOKEN spelt as a null-terminated string. The string is
2447 freed when the reader is destroyed. Useful for diagnostics. */
2449 cpp_token_as_text (cpp_reader *pfile, const cpp_token *token)
2451 unsigned int len = cpp_token_len (token) + 1;
2452 unsigned char *start = _cpp_unaligned_alloc (pfile, len), *end;
2454 end = cpp_spell_token (pfile, token, start, false);
2460 /* Returns a pointer to a string which spells the token defined by
2461 TYPE and FLAGS. Used by C front ends, which really should move to
2462 using cpp_token_as_text. */
2464 cpp_type2name (enum cpp_ttype type, unsigned char flags)
2466 if (flags & DIGRAPH)
2467 return (const char *) cpp_digraph2name (type);
2468 else if (flags & NAMED_OP)
2469 return cpp_named_operator2name (type);
2471 return (const char *) token_spellings[type].name;
2474 /* Writes the spelling of token to FP, without any preceding space.
2475 Separated from cpp_spell_token for efficiency - to avoid stdio
2476 double-buffering. */
2478 cpp_output_token (const cpp_token *token, FILE *fp)
2480 switch (TOKEN_SPELL (token))
2482 case SPELL_OPERATOR:
2484 const unsigned char *spelling;
2487 if (token->flags & DIGRAPH)
2488 spelling = cpp_digraph2name (token->type);
2489 else if (token->flags & NAMED_OP)
2492 spelling = TOKEN_NAME (token);
2497 while ((c = *++spelling) != '\0');
2505 const unsigned char * name = NODE_NAME (token->val.node.node);
2507 for (i = 0; i < NODE_LEN (token->val.node.node); i++)
2508 if (name[i] & ~0x7F)
2510 unsigned char buffer[10];
2511 i += utf8_to_ucn (buffer, name + i) - 1;
2512 fwrite (buffer, 1, 10, fp);
2515 fputc (NODE_NAME (token->val.node.node)[i], fp);
2520 fwrite (token->val.str.text, 1, token->val.str.len, fp);
2524 /* An error, most probably. */
2529 /* Compare two tokens. */
2531 _cpp_equiv_tokens (const cpp_token *a, const cpp_token *b)
2533 if (a->type == b->type && a->flags == b->flags)
2534 switch (TOKEN_SPELL (a))
2536 default: /* Keep compiler happy. */
2537 case SPELL_OPERATOR:
2538 /* token_no is used to track where multiple consecutive ##
2539 tokens were originally located. */
2540 return (a->type != CPP_PASTE || a->val.token_no == b->val.token_no);
2542 return (a->type != CPP_MACRO_ARG
2543 || a->val.macro_arg.arg_no == b->val.macro_arg.arg_no);
2545 return a->val.node.node == b->val.node.node;
2547 return (a->val.str.len == b->val.str.len
2548 && !memcmp (a->val.str.text, b->val.str.text,
2555 /* Returns nonzero if a space should be inserted to avoid an
2556 accidental token paste for output. For simplicity, it is
2557 conservative, and occasionally advises a space where one is not
2558 needed, e.g. "." and ".2". */
2560 cpp_avoid_paste (cpp_reader *pfile, const cpp_token *token1,
2561 const cpp_token *token2)
2563 enum cpp_ttype a = token1->type, b = token2->type;
2566 if (token1->flags & NAMED_OP)
2568 if (token2->flags & NAMED_OP)
2572 if (token2->flags & DIGRAPH)
2573 c = digraph_spellings[(int) b - (int) CPP_FIRST_DIGRAPH][0];
2574 else if (token_spellings[b].category == SPELL_OPERATOR)
2575 c = token_spellings[b].name[0];
2577 /* Quickly get everything that can paste with an '='. */
2578 if ((int) a <= (int) CPP_LAST_EQ && c == '=')
2583 case CPP_GREATER: return c == '>';
2584 case CPP_LESS: return c == '<' || c == '%' || c == ':';
2585 case CPP_PLUS: return c == '+';
2586 case CPP_MINUS: return c == '-' || c == '>';
2587 case CPP_DIV: return c == '/' || c == '*'; /* Comments. */
2588 case CPP_MOD: return c == ':' || c == '>';
2589 case CPP_AND: return c == '&';
2590 case CPP_OR: return c == '|';
2591 case CPP_COLON: return c == ':' || c == '>';
2592 case CPP_DEREF: return c == '*';
2593 case CPP_DOT: return c == '.' || c == '%' || b == CPP_NUMBER;
2594 case CPP_HASH: return c == '#' || c == '%'; /* Digraph form. */
2595 case CPP_NAME: return ((b == CPP_NUMBER
2596 && name_p (pfile, &token2->val.str))
2598 || b == CPP_CHAR || b == CPP_STRING); /* L */
2599 case CPP_NUMBER: return (b == CPP_NUMBER || b == CPP_NAME
2600 || c == '.' || c == '+' || c == '-');
2602 case CPP_OTHER: return ((token1->val.str.text[0] == '\\'
2604 || (CPP_OPTION (pfile, objc)
2605 && token1->val.str.text[0] == '@'
2606 && (b == CPP_NAME || b == CPP_STRING)));
2613 /* Output all the remaining tokens on the current line, and a newline
2614 character, to FP. Leading whitespace is removed. If there are
2615 macros, special token padding is not performed. */
2617 cpp_output_line (cpp_reader *pfile, FILE *fp)
2619 const cpp_token *token;
2621 token = cpp_get_token (pfile);
2622 while (token->type != CPP_EOF)
2624 cpp_output_token (token, fp);
2625 token = cpp_get_token (pfile);
2626 if (token->flags & PREV_WHITE)
2633 /* Return a string representation of all the remaining tokens on the
2634 current line. The result is allocated using xmalloc and must be
2635 freed by the caller. */
2637 cpp_output_line_to_string (cpp_reader *pfile, const unsigned char *dir_name)
2639 const cpp_token *token;
2640 unsigned int out = dir_name ? ustrlen (dir_name) : 0;
2641 unsigned int alloced = 120 + out;
2642 unsigned char *result = (unsigned char *) xmalloc (alloced);
2644 /* If DIR_NAME is empty, there are no initial contents. */
2647 sprintf ((char *) result, "#%s ", dir_name);
2651 token = cpp_get_token (pfile);
2652 while (token->type != CPP_EOF)
2654 unsigned char *last;
2655 /* Include room for a possible space and the terminating nul. */
2656 unsigned int len = cpp_token_len (token) + 2;
2658 if (out + len > alloced)
2661 if (out + len > alloced)
2662 alloced = out + len;
2663 result = (unsigned char *) xrealloc (result, alloced);
2666 last = cpp_spell_token (pfile, token, &result[out], 0);
2667 out = last - result;
2669 token = cpp_get_token (pfile);
2670 if (token->flags & PREV_WHITE)
2671 result[out++] = ' ';
2678 /* Memory buffers. Changing these three constants can have a dramatic
2679 effect on performance. The values here are reasonable defaults,
2680 but might be tuned. If you adjust them, be sure to test across a
2681 range of uses of cpplib, including heavy nested function-like macro
2682 expansion. Also check the change in peak memory usage (NJAMD is a
2683 good tool for this). */
2684 #define MIN_BUFF_SIZE 8000
2685 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
2686 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
2687 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
2689 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
2690 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
2693 /* Create a new allocation buffer. Place the control block at the end
2694 of the buffer, so that buffer overflows will cause immediate chaos. */
2696 new_buff (size_t len)
2699 unsigned char *base;
2701 if (len < MIN_BUFF_SIZE)
2702 len = MIN_BUFF_SIZE;
2703 len = CPP_ALIGN (len);
2705 base = XNEWVEC (unsigned char, len + sizeof (_cpp_buff));
2706 result = (_cpp_buff *) (base + len);
2707 result->base = base;
2709 result->limit = base + len;
2710 result->next = NULL;
2714 /* Place a chain of unwanted allocation buffers on the free list. */
2716 _cpp_release_buff (cpp_reader *pfile, _cpp_buff *buff)
2718 _cpp_buff *end = buff;
2722 end->next = pfile->free_buffs;
2723 pfile->free_buffs = buff;
2726 /* Return a free buffer of size at least MIN_SIZE. */
2728 _cpp_get_buff (cpp_reader *pfile, size_t min_size)
2730 _cpp_buff *result, **p;
2732 for (p = &pfile->free_buffs;; p = &(*p)->next)
2737 return new_buff (min_size);
2739 size = result->limit - result->base;
2740 /* Return a buffer that's big enough, but don't waste one that's
2742 if (size >= min_size && size <= BUFF_SIZE_UPPER_BOUND (min_size))
2747 result->next = NULL;
2748 result->cur = result->base;
2752 /* Creates a new buffer with enough space to hold the uncommitted
2753 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies
2754 the excess bytes to the new buffer. Chains the new buffer after
2755 BUFF, and returns the new buffer. */
2757 _cpp_append_extend_buff (cpp_reader *pfile, _cpp_buff *buff, size_t min_extra)
2759 size_t size = EXTENDED_BUFF_SIZE (buff, min_extra);
2760 _cpp_buff *new_buff = _cpp_get_buff (pfile, size);
2762 buff->next = new_buff;
2763 memcpy (new_buff->base, buff->cur, BUFF_ROOM (buff));
2767 /* Creates a new buffer with enough space to hold the uncommitted
2768 remaining bytes of the buffer pointed to by BUFF, and at least
2769 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer.
2770 Chains the new buffer before the buffer pointed to by BUFF, and
2771 updates the pointer to point to the new buffer. */
2773 _cpp_extend_buff (cpp_reader *pfile, _cpp_buff **pbuff, size_t min_extra)
2775 _cpp_buff *new_buff, *old_buff = *pbuff;
2776 size_t size = EXTENDED_BUFF_SIZE (old_buff, min_extra);
2778 new_buff = _cpp_get_buff (pfile, size);
2779 memcpy (new_buff->base, old_buff->cur, BUFF_ROOM (old_buff));
2780 new_buff->next = old_buff;
2784 /* Free a chain of buffers starting at BUFF. */
2786 _cpp_free_buff (_cpp_buff *buff)
2790 for (; buff; buff = next)
2797 /* Allocate permanent, unaligned storage of length LEN. */
2799 _cpp_unaligned_alloc (cpp_reader *pfile, size_t len)
2801 _cpp_buff *buff = pfile->u_buff;
2802 unsigned char *result = buff->cur;
2804 if (len > (size_t) (buff->limit - result))
2806 buff = _cpp_get_buff (pfile, len);
2807 buff->next = pfile->u_buff;
2808 pfile->u_buff = buff;
2812 buff->cur = result + len;
2816 /* Allocate permanent, unaligned storage of length LEN from a_buff.
2817 That buffer is used for growing allocations when saving macro
2818 replacement lists in a #define, and when parsing an answer to an
2819 assertion in #assert, #unassert or #if (and therefore possibly
2820 whilst expanding macros). It therefore must not be used by any
2821 code that they might call: specifically the lexer and the guts of
2824 All existing other uses clearly fit this restriction: storing
2825 registered pragmas during initialization. */
2827 _cpp_aligned_alloc (cpp_reader *pfile, size_t len)
2829 _cpp_buff *buff = pfile->a_buff;
2830 unsigned char *result = buff->cur;
2832 if (len > (size_t) (buff->limit - result))
2834 buff = _cpp_get_buff (pfile, len);
2835 buff->next = pfile->a_buff;
2836 pfile->a_buff = buff;
2840 buff->cur = result + len;
2844 /* Say which field of TOK is in use. */
2846 enum cpp_token_fld_kind
2847 cpp_token_val_index (cpp_token *tok)
2849 switch (TOKEN_SPELL (tok))
2852 return CPP_TOKEN_FLD_NODE;
2854 return CPP_TOKEN_FLD_STR;
2855 case SPELL_OPERATOR:
2856 if (tok->type == CPP_PASTE)
2857 return CPP_TOKEN_FLD_TOKEN_NO;
2859 return CPP_TOKEN_FLD_NONE;
2861 if (tok->type == CPP_MACRO_ARG)
2862 return CPP_TOKEN_FLD_ARG_NO;
2863 else if (tok->type == CPP_PADDING)
2864 return CPP_TOKEN_FLD_SOURCE;
2865 else if (tok->type == CPP_PRAGMA)
2866 return CPP_TOKEN_FLD_PRAGMA;
2867 /* else fall through */
2869 return CPP_TOKEN_FLD_NONE;
2873 /* All tokens lexed in R after calling this function will be forced to have
2874 their source_location the same as the location referenced by P, until
2875 cpp_stop_forcing_token_locations is called for R. */
2878 cpp_force_token_locations (cpp_reader *r, source_location *p)
2880 r->forced_token_location_p = p;
2883 /* Go back to assigning locations naturally for lexed tokens. */
2886 cpp_stop_forcing_token_locations (cpp_reader *r)
2888 r->forced_token_location_p = NULL;