1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000 Free Software Foundation, Inc.
3 Contributed by Per Bothner, 1994-95.
4 Based on CCCP program by Paul Rubin, June 1986
5 Adapted to ANSI C, Richard Stallman, Jan 1987
6 Broken out to separate file, Zack Weinberg, Mar 2000
7 Single-pass line tokenization by Neil Booth, April 2000
9 This program is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 2, or (at your option) any
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
27 o Check line numbers assigned to all errors.
28 o Replace strncmp with memcmp almost everywhere.
29 o lex_line's use of cur_token, flags and list->token_used is a bit opaque.
30 o Distinguish integers, floats, and 'other' pp-numbers.
31 o Store ints and char constants as binary values.
32 o New command-line assertion syntax.
33 o Work towards functions in cpperror.c taking a message level parameter.
34 If we do this, merge the common code of do_warning and do_error.
35 o Comment all functions, and describe macro expansion algorithm.
36 o Move as much out of header files as possible.
37 o Remove single quote pairs `', and some '', from diagnostics.
38 o Correct pastability test for CPP_NAME and CPP_NUMBER.
49 static const cpp_token placemarker_token = {0, 0, CPP_PLACEMARKER, 0 UNION_INIT_ZERO};
50 static const cpp_token eof_token = {0, 0, CPP_EOF, 0 UNION_INIT_ZERO};
52 /* Flags for cpp_context. */
53 #define CONTEXT_PASTEL (1 << 0) /* An argument context on LHS of ##. */
54 #define CONTEXT_PASTER (1 << 1) /* An argument context on RHS of ##. */
55 #define CONTEXT_RAW (1 << 2) /* If argument tokens already expanded. */
56 #define CONTEXT_ARG (1 << 3) /* If an argument context. */
58 typedef struct cpp_context cpp_context;
63 const cpp_toklist *list; /* Used for macro contexts only. */
64 const cpp_token **arg; /* Used for arg contexts only. */
67 /* Pushed token to be returned by next call to get_raw_token. */
68 const cpp_token *pushed_token;
70 struct macro_args *args; /* The arguments for a function-like
71 macro. NULL otherwise. */
72 unsigned short posn; /* Current posn, index into u. */
73 unsigned short count; /* No. of tokens in u. */
78 typedef struct macro_args macro_args;
82 const cpp_token **tokens;
83 unsigned int capacity;
88 static const cpp_token *get_raw_token PARAMS ((cpp_reader *));
89 static const cpp_token *parse_arg PARAMS ((cpp_reader *, int, unsigned int,
90 macro_args *, unsigned int *));
91 static int parse_args PARAMS ((cpp_reader *, cpp_hashnode *, macro_args *));
92 static void save_token PARAMS ((macro_args *, const cpp_token *));
93 static int pop_context PARAMS ((cpp_reader *));
94 static int push_macro_context PARAMS ((cpp_reader *, const cpp_token *));
95 static void push_arg_context PARAMS ((cpp_reader *, const cpp_token *));
96 static void free_macro_args PARAMS ((macro_args *));
98 #define auto_expand_name_space(list) \
99 _cpp_expand_name_space ((list), 1 + (list)->name_cap / 2)
100 static void dump_param_spelling PARAMS ((FILE *, const cpp_toklist *,
102 static void output_line_command PARAMS ((cpp_reader *, cpp_printer *,
105 static void process_directive PARAMS ((cpp_reader *, const cpp_token *));
106 static unsigned char *trigraph_replace PARAMS ((cpp_reader *, unsigned char *,
108 static const unsigned char *backslash_start PARAMS ((cpp_reader *,
109 const unsigned char *));
110 static int skip_block_comment PARAMS ((cpp_reader *));
111 static int skip_line_comment PARAMS ((cpp_reader *));
112 static void adjust_column PARAMS ((cpp_reader *, const U_CHAR *));
113 static void skip_whitespace PARAMS ((cpp_reader *, int));
114 static const U_CHAR *parse_name PARAMS ((cpp_reader *, cpp_token *,
115 const U_CHAR *, const U_CHAR *));
116 static void parse_number PARAMS ((cpp_reader *, cpp_toklist *, cpp_string *));
117 static void parse_string PARAMS ((cpp_reader *, cpp_toklist *, cpp_token *,
119 static int trigraph_ok PARAMS ((cpp_reader *, const unsigned char *));
120 static void save_comment PARAMS ((cpp_toklist *, cpp_token *,
121 const unsigned char *,
122 unsigned int, unsigned int));
123 static void lex_line PARAMS ((cpp_reader *, cpp_toklist *));
124 static int lex_next PARAMS ((cpp_reader *, int));
125 static int is_macro_disabled PARAMS ((cpp_reader *, const cpp_toklist *,
128 static cpp_token *stringify_arg PARAMS ((cpp_reader *, const cpp_token *));
129 static void expand_context_stack PARAMS ((cpp_reader *));
130 static unsigned char * spell_token PARAMS ((cpp_reader *, const cpp_token *,
132 static void output_token PARAMS ((cpp_reader *, FILE *, const cpp_token *,
133 const cpp_token *, int));
134 typedef unsigned int (* speller) PARAMS ((unsigned char *, cpp_toklist *,
136 static cpp_token *make_string_token PARAMS ((cpp_token *, const U_CHAR *,
138 static cpp_token *alloc_number_token PARAMS ((cpp_reader *, int number));
139 static const cpp_token *special_symbol PARAMS ((cpp_reader *, cpp_hashnode *,
141 static cpp_token *duplicate_token PARAMS ((cpp_reader *, const cpp_token *));
142 static const cpp_token *maybe_paste_with_next PARAMS ((cpp_reader *,
144 static enum cpp_ttype can_paste PARAMS ((cpp_reader *, const cpp_token *,
145 const cpp_token *, int *));
146 static unsigned int prevent_macro_expansion PARAMS ((cpp_reader *));
147 static void restore_macro_expansion PARAMS ((cpp_reader *, unsigned int));
148 static cpp_token *get_temp_token PARAMS ((cpp_reader *));
149 static void release_temp_tokens PARAMS ((cpp_reader *));
150 static U_CHAR * quote_string PARAMS ((U_CHAR *, const U_CHAR *, unsigned int));
151 static void process_directive PARAMS ((cpp_reader *, const cpp_token *));
153 #define INIT_TOKEN_STR(list, token) \
154 do {(token)->val.str.len = 0; \
155 (token)->val.str.text = (list)->namebuf + (list)->name_used; \
158 #define VALID_SIGN(c, prevc) \
159 (((c) == '+' || (c) == '-') && \
160 ((prevc) == 'e' || (prevc) == 'E' \
161 || (((prevc) == 'p' || (prevc) == 'P') && !CPP_OPTION (pfile, c89))))
163 /* Handle LF, CR, CR-LF and LF-CR style newlines. Assumes next
164 character, if any, is in buffer. */
166 #define handle_newline(cur, limit, c) \
168 if ((cur) < (limit) && *(cur) == '\r' + '\n' - c) \
170 pfile->buffer->lineno++; \
171 pfile->buffer->line_base = (cur); \
172 pfile->col_adjust = 0; \
175 #define IMMED_TOKEN() (!(cur_token->flags & PREV_WHITE))
176 #define PREV_TOKEN_TYPE (cur_token[-1].type)
178 #define PUSH_TOKEN(ttype) cur_token++->type = (ttype)
179 #define REVISE_TOKEN(ttype) cur_token[-1].type = (ttype)
180 #define BACKUP_TOKEN(ttype) (--cur_token)->type = (ttype)
181 #define BACKUP_DIGRAPH(ttype) do { \
182 BACKUP_TOKEN(ttype); cur_token->flags |= DIGRAPH;} while (0)
184 /* An upper bound on the number of bytes needed to spell a token,
185 including preceding whitespace. */
186 static inline size_t TOKEN_LEN PARAMS ((const cpp_token *));
189 const cpp_token *token;
193 switch (TOKEN_SPELL (token))
195 default: len = 0; break;
196 case SPELL_STRING: len = token->val.str.len; break;
197 case SPELL_IDENT: len = token->val.node->length; break;
202 #define IS_ARG_CONTEXT(c) ((c)->flags & CONTEXT_ARG)
203 #define CURRENT_CONTEXT(pfile) ((pfile)->contexts + (pfile)->cur_context)
204 #define ON_REST_ARG(c) \
205 (((c)->flags & VAR_ARGS) \
206 && (c)->u.list->tokens[(c)->posn].val.aux \
207 == (unsigned int) ((c)->u.list->paramc - 1))
209 #define ASSIGN_FLAGS_AND_POS(d, s) \
210 do {(d)->flags = (s)->flags & (PREV_WHITE | BOL | PASTE_LEFT); \
211 if ((d)->flags & BOL) {(d)->col = (s)->col; (d)->line = (s)->line;} \
214 /* f is flags, just consisting of PREV_WHITE | BOL. */
215 #define MODIFY_FLAGS_AND_POS(d, s, f) \
216 do {(d)->flags &= ~(PREV_WHITE | BOL); (d)->flags |= (f); \
217 if ((f) & BOL) {(d)->col = (s)->col; (d)->line = (s)->line;} \
220 #define OP(e, s) { SPELL_OPERATOR, U s },
221 #define TK(e, s) { s, U STRINGX (e) },
223 const struct token_spelling
224 _cpp_token_spellings [N_TTYPES] = {TTYPE_TABLE };
229 /* The following table is used by trigraph_ok/trigraph_replace. If we
230 have designated initializers, it can be constant data; otherwise,
231 it is set up at runtime by _cpp_init_input_buffer. */
233 #if (GCC_VERSION >= 2007)
234 #define init_trigraph_map() /* nothing */
235 #define TRIGRAPH_MAP \
236 __extension__ static const U_CHAR trigraph_map[UCHAR_MAX + 1] = {
238 #define s(p, v) [p] = v,
240 #define TRIGRAPH_MAP static U_CHAR trigraph_map[UCHAR_MAX + 1] = { 0 }; \
241 static void init_trigraph_map PARAMS ((void)) { \
242 unsigned char *x = trigraph_map;
244 #define s(p, v) x[p] = v;
248 s('=', '#') s(')', ']') s('!', '|')
249 s('(', '[') s('\'', '^') s('>', '}')
250 s('/', '\\') s('<', '{') s('-', '~')
257 /* Notify the compiler proper that the current line number has jumped,
258 or the current file name has changed. */
261 output_line_command (pfile, print, line)
266 cpp_buffer *ip = CPP_BUFFER (pfile);
271 /* End the previous line of text. */
272 if (pfile->need_newline)
274 putc ('\n', print->outf);
277 pfile->need_newline = 0;
279 if (CPP_OPTION (pfile, no_line_commands))
282 /* If the current file has not changed, we can output a few newlines
283 instead if we want to increase the line number by a small amount.
284 We cannot do this if print->lineno is zero, because that means we
285 haven't output any line commands yet. (The very first line
286 command output is a `same_file' command.)
288 'nominal_fname' values are unique, so they can be compared by
289 comparing pointers. */
290 if (ip->nominal_fname == print->last_fname && print->lineno > 0
291 && line >= print->lineno && line < print->lineno + 8)
293 while (line > print->lineno)
295 putc ('\n', print->outf);
301 fprintf (print->outf, "# %u \"%s\"%s\n", line, ip->nominal_fname,
302 cpp_syshdr_flags (pfile, ip));
304 print->last_fname = ip->nominal_fname;
305 print->lineno = line;
308 /* Like fprintf, but writes to a printer object. You should be sure
309 always to generate a complete line when you use this function. */
311 cpp_printf VPARAMS ((cpp_reader *pfile, cpp_printer *print,
312 const char *fmt, ...))
315 #ifndef ANSI_PROTOTYPES
323 #ifndef ANSI_PROTOTYPES
324 pfile = va_arg (ap, cpp_reader *);
325 print = va_arg (ap, cpp_printer *);
326 fmt = va_arg (ap, const char *);
329 /* End the previous line of text. */
330 if (pfile->need_newline)
331 putc ('\n', print->outf);
332 pfile->need_newline = 0;
334 vfprintf (print->outf, fmt, ap);
338 /* Scan until CPP_BUFFER (PFILE) is exhausted, discarding output. */
341 cpp_scan_buffer_nooutput (pfile)
344 cpp_buffer *stop = CPP_PREV_BUFFER (CPP_BUFFER (pfile));
345 const cpp_token *token;
347 /* In no-output mode, we can ignore everything but directives. */
350 token = _cpp_get_token (pfile);
352 if (token->type == CPP_EOF)
354 cpp_pop_buffer (pfile);
355 if (CPP_BUFFER (pfile) == stop)
359 if (token->type == CPP_HASH && token->flags & BOL
360 && pfile->token_list.directive)
362 process_directive (pfile, token);
366 _cpp_skip_rest_of_line (pfile);
370 /* Scan until CPP_BUFFER (pfile) is exhausted, writing output to PRINT. */
372 cpp_scan_buffer (pfile, print)
376 cpp_buffer *stop = CPP_PREV_BUFFER (CPP_BUFFER (pfile));
377 const cpp_token *token, *prev = 0;
381 token = _cpp_get_token (pfile);
382 if (token->type == CPP_EOF)
384 cpp_pop_buffer (pfile);
386 if (CPP_BUFFER (pfile) == stop)
393 if (token->flags & BOL)
395 if (token->type == CPP_HASH && pfile->token_list.directive)
397 process_directive (pfile, token);
401 output_line_command (pfile, print, pfile->token_list.line);
405 if (token->type != CPP_PLACEMARKER)
407 output_token (pfile, print->outf, token, prev, 1);
408 pfile->need_newline = 1;
415 /* Helper routine used by parse_include, which can't see spell_token.
416 Reinterpret the current line as an h-char-sequence (< ... >); we are
417 looking at the first token after the <. */
419 _cpp_glue_header_name (pfile)
429 buf = xmalloc (avail);
433 t = _cpp_get_token (pfile);
434 if (t->type == CPP_GREATER || t->type == CPP_EOF)
437 if (len + TOKEN_LEN (t) > avail)
439 avail = len + TOKEN_LEN (t) + 40;
440 buf = xrealloc (buf, avail);
443 if (t->flags & PREV_WHITE)
446 p = spell_token (pfile, t, buf + len);
447 len = (size_t) (p - buf); /* p known >= buf */
450 if (t->type == CPP_EOF)
451 cpp_error (pfile, "missing terminating > character");
453 buf = xrealloc (buf, len);
455 hdr = get_temp_token (pfile);
456 hdr->type = CPP_HEADER_NAME;
458 hdr->val.str.text = buf;
459 hdr->val.str.len = len;
463 /* Token-buffer helper functions. */
465 /* Expand a token list's string space. It is *vital* that
466 list->tokens_used is correct, to get pointer fix-up right. */
468 _cpp_expand_name_space (list, len)
472 const U_CHAR *old_namebuf;
474 old_namebuf = list->namebuf;
475 list->name_cap += len;
476 list->namebuf = (unsigned char *) xrealloc (list->namebuf, list->name_cap);
478 /* Fix up token text pointers. */
479 if (list->namebuf != old_namebuf)
483 for (i = 0; i < list->tokens_used; i++)
484 if (TOKEN_SPELL (&list->tokens[i]) == SPELL_STRING)
485 list->tokens[i].val.str.text += (list->namebuf - old_namebuf);
489 /* If there is not enough room for LEN more characters, expand the
490 list by just enough to have room for LEN characters. */
492 _cpp_reserve_name_space (list, len)
496 unsigned int room = list->name_cap - list->name_used;
499 _cpp_expand_name_space (list, len - room);
502 /* Expand the number of tokens in a list. */
504 _cpp_expand_token_space (list, count)
510 list->tokens_cap += count;
511 n = list->tokens_cap;
512 if (list->flags & LIST_OFFSET)
514 list->tokens = (cpp_token *)
515 xrealloc (list->tokens, n * sizeof (cpp_token));
516 if (list->flags & LIST_OFFSET)
517 list->tokens++; /* Skip the dummy. */
520 /* Initialize a token list. If flags is DUMMY_TOKEN, we allocate
521 an extra token in front of the token list, as this allows the lexer
522 to always peek at the previous token without worrying about
523 underflowing the list, and some initial space. Otherwise, no
524 token- or name-space is allocated, and there is no dummy token. */
526 _cpp_init_toklist (list, flags)
530 if (flags == NO_DUMMY_TOKEN)
532 list->tokens_cap = 0;
540 /* Initialize token space. Put a dummy token before the start
541 that will fail matches. */
542 list->tokens_cap = 256; /* 4K's worth. */
543 list->tokens = (cpp_token *)
544 xmalloc ((list->tokens_cap + 1) * sizeof (cpp_token));
545 list->tokens[0].type = CPP_EOF;
548 /* Initialize name space. */
549 list->name_cap = 1024;
550 list->namebuf = (unsigned char *) xmalloc (list->name_cap);
551 list->flags = LIST_OFFSET;
554 _cpp_clear_toklist (list);
557 /* Clear a token list. */
559 _cpp_clear_toklist (list)
562 list->tokens_used = 0;
566 list->params_len = 0;
567 list->flags &= LIST_OFFSET; /* clear all but that one */
570 /* Free a token list. Does not free the list itself, which may be
571 embedded in a larger structure. */
573 _cpp_free_toklist (list)
574 const cpp_toklist *list;
576 if (list->flags & LIST_OFFSET)
577 free (list->tokens - 1); /* Backup over dummy token. */
580 free (list->namebuf);
583 /* Compare two tokens. */
585 _cpp_equiv_tokens (a, b)
586 const cpp_token *a, *b;
588 if (a->type == b->type && a->flags == b->flags)
589 switch (TOKEN_SPELL (a))
591 default: /* Keep compiler happy. */
596 return a->val.aux == b->val.aux; /* arg_no or character. */
598 return a->val.node == b->val.node;
600 return (a->val.str.len == b->val.str.len
601 && !memcmp (a->val.str.text, b->val.str.text,
608 /* Compare two token lists. */
610 _cpp_equiv_toklists (a, b)
611 const cpp_toklist *a, *b;
615 if (a->tokens_used != b->tokens_used
616 || a->flags != b->flags
617 || a->paramc != b->paramc)
620 for (i = 0; i < a->tokens_used; i++)
621 if (! _cpp_equiv_tokens (&a->tokens[i], &b->tokens[i]))
628 Compares, the token TOKEN to the NUL-terminated string STRING.
629 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
632 cpp_ideq (token, string)
633 const cpp_token *token;
636 if (token->type != CPP_NAME)
639 return !ustrcmp (token->val.node->name, (const U_CHAR *)string);
644 The original lexer in cpplib was made up of two passes: a first pass
645 that replaced trigraphs and deleted esacped newlines, and a second
646 pass that tokenized the result of the first pass. Tokenisation was
647 performed by peeking at the next character in the input stream. For
648 example, if the input stream contained "!=", the handler for the !
649 character would peek at the next character, and if it were a '='
650 would skip over it, and return a "!=" token, otherwise it would
651 return just the "!" token.
653 To implement a single-pass lexer, this peeking ahead is unworkable.
654 An arbitrary number of escaped newlines, and trigraphs (in particular
655 ??/ which translates to the escape \), could separate the '!' and '='
656 in the input stream, yet the next token is still a "!=".
658 Suppose instead that we lex by one logical line at a time, producing
659 a token list or stack for each logical line, and when seeing the '!'
660 push a CPP_NOT token on the list. Then if the '!' is part of a
661 longer token ("!=") we know we must see the remainder of the token by
662 the time we reach the end of the logical line. Thus we can have the
663 '=' handler look at the previous token (at the end of the list / top
664 of the stack) and see if it is a "!" token, and if so, instead of
665 pushing a "=" token revise the existing token to be a "!=" token.
667 This works in the presence of escaped newlines, because the '\' would
668 have been pushed on the top of the stack as a CPP_BACKSLASH. The
669 newline ('\n' or '\r') handler looks at the token at the top of the
670 stack to see if it is a CPP_BACKSLASH, and if so discards both.
671 Hence the '=' handler would never see any intervening tokens.
673 To make trigraphs work in this context, as in precedence trigraphs
674 are highest and converted before anything else, the '?' handler does
675 lookahead to see if it is a trigraph, and if so skips the trigraph
676 and pushes the token it represents onto the top of the stack. This
677 also works in the particular case of a CPP_BACKSLASH trigraph.
679 To the preprocessor, whitespace is only significant to the point of
680 knowing whether whitespace precedes a particular token. For example,
681 the '=' handler needs to know whether there was whitespace between it
682 and a "!" token on the top of the stack, to make the token conversion
683 decision correctly. So each token has a PREV_WHITE flag to
684 indicate this - the standard permits consecutive whitespace to be
685 regarded as a single space. The compiler front ends are not
686 interested in whitespace at all; they just require a token stream.
687 Another place where whitespace is significant to the preprocessor is
688 a #define statment - if there is whitespace between the macro name
689 and an initial "(" token the macro is "object-like", otherwise it is
690 a function-like macro that takes arguments.
692 However, all is not rosy. Parsing of identifiers, numbers, comments
693 and strings becomes trickier because of the possibility of raw
694 trigraphs and escaped newlines in the input stream.
696 The trigraphs are three consecutive characters beginning with two
697 question marks. A question mark is not valid as part of a number or
698 identifier, so parsing of a number or identifier terminates normally
699 upon reaching it, returning to the mainloop which handles the
700 trigraph just like it would in any other position. Similarly for the
701 backslash of a backslash-newline combination. So we just need the
702 escaped-newline dropper in the mainloop to check if the token on the
703 top of the stack after dropping the escaped newline is a number or
704 identifier, and if so to continue the processing it as if nothing had
707 For strings, we replace trigraphs whenever we reach a quote or
708 newline, because there might be a backslash trigraph escaping them.
709 We need to be careful that we start trigraph replacing from where we
710 left off previously, because it is possible for a first scan to leave
711 "fake" trigraphs that a second scan would pick up as real (e.g. the
712 sequence "????/\n=" would find a fake ??= trigraph after removing the
715 For line comments, on reaching a newline we scan the previous
716 character(s) to see if it escaped, and continue if it is. Block
717 comments ignore everything and just focus on finding the comment
718 termination mark. The only difficult thing, and it is surprisingly
719 tricky, is checking if an asterisk precedes the final slash since
720 they could be separated by escaped newlines. If the preprocessor is
721 invoked with the output comments option, we don't bother removing
722 escaped newlines and replacing trigraphs for output.
724 Finally, numbers can begin with a period, which is pushed initially
725 as a CPP_DOT token in its own right. The digit handler checks if the
726 previous token was a CPP_DOT not separated by whitespace, and if so
727 pops it off the stack and pushes a period into the number's buffer
728 before calling the number parser.
732 static const unsigned char *digraph_spellings [] = {U"%:", U"%:%:", U"<:",
733 U":>", U"<%", U"%>"};
735 /* Call when a trigraph is encountered. It warns if necessary, and
736 returns true if the trigraph should be honoured. END is the third
737 character of a trigraph in the input stream. */
739 trigraph_ok (pfile, end)
741 const unsigned char *end;
743 int accept = CPP_OPTION (pfile, trigraphs);
745 if (CPP_OPTION (pfile, warn_trigraphs))
747 unsigned int col = end - 1 - pfile->buffer->line_base;
749 cpp_warning_with_line (pfile, pfile->buffer->lineno, col,
750 "trigraph ??%c converted to %c",
751 (int) *end, (int) trigraph_map[*end]);
753 cpp_warning_with_line (pfile, pfile->buffer->lineno, col,
754 "trigraph ??%c ignored", (int) *end);
759 /* Scan a string for trigraphs, warning or replacing them inline as
760 appropriate. When parsing a string, we must call this routine
761 before processing a newline character (if trigraphs are enabled),
762 since the newline might be escaped by a preceding backslash
763 trigraph sequence. Returns a pointer to the end of the name after
766 static unsigned char *
767 trigraph_replace (pfile, src, limit)
770 unsigned char *limit;
774 /* Starting with src[1], find two consecutive '?'. The case of no
775 trigraphs is streamlined. */
777 for (src++; src + 1 < limit; src += 2)
782 /* Make src point to the 1st (NOT 2nd) of two consecutive '?'s. */
785 else if (src + 2 == limit || src[1] != '?')
788 /* Check if it really is a trigraph. */
789 if (trigraph_map[src[2]] == 0)
797 /* Now we have a trigraph, we need to scan the remaining buffer, and
798 copy-shifting its contents left if replacement is enabled. */
799 for (; src + 2 < limit; dest++, src++)
800 if ((*dest = *src) == '?' && src[1] == '?' && trigraph_map[src[2]])
804 if (trigraph_ok (pfile, pfile->buffer->cur - (limit - src)))
805 *dest = trigraph_map[*src];
808 /* Copy remaining (at most 2) characters. */
814 /* If CUR is a backslash or the end of a trigraphed backslash, return
815 a pointer to its beginning, otherwise NULL. We don't read beyond
816 the buffer start, because there is the start of the comment in the
818 static const unsigned char *
819 backslash_start (pfile, cur)
821 const unsigned char *cur;
825 if (cur[0] == '/' && cur[-1] == '?' && cur[-2] == '?'
826 && trigraph_ok (pfile, cur))
831 /* Skip a C-style block comment. This is probably the trickiest
832 handler. We find the end of the comment by seeing if an asterisk
833 is before every '/' we encounter. The nasty complication is that a
834 previous asterisk may be separated by one or more escaped newlines.
835 Returns non-zero if comment terminated by EOF, zero otherwise. */
837 skip_block_comment (pfile)
840 cpp_buffer *buffer = pfile->buffer;
841 const unsigned char *char_after_star = 0;
842 const unsigned char *cur = buffer->cur;
844 for (; cur < buffer->rlimit; )
846 unsigned char c = *cur++;
848 /* People like decorating comments with '*', so check for
849 '/' instead for efficiency. */
852 /* Don't view / then * then / as finishing the comment. */
853 if ((cur[-2] == '*' && cur - 1 > buffer->cur)
854 || cur - 1 == char_after_star)
860 /* Warn about potential nested comments, but not when
861 the final character inside the comment is a '/'.
862 Don't bother to get it right across escaped newlines. */
863 if (CPP_OPTION (pfile, warn_comments) && cur + 1 < buffer->rlimit
864 && cur[0] == '*' && cur[1] != '/')
867 cpp_warning (pfile, "'/*' within comment");
870 else if (is_vspace (c))
872 const unsigned char* bslash = backslash_start (pfile, cur - 2);
874 handle_newline (cur, buffer->rlimit, c);
875 /* Work correctly if there is an asterisk before an
876 arbirtrarily long sequence of escaped newlines. */
877 if (bslash && (bslash[-1] == '*' || bslash == char_after_star))
878 char_after_star = cur;
883 adjust_column (pfile, cur - 1);
890 /* Skip a C++ line comment. Handles escaped newlines. Returns
891 non-zero if a multiline comment. */
893 skip_line_comment (pfile)
896 cpp_buffer *buffer = pfile->buffer;
897 register const unsigned char *cur = buffer->cur;
900 for (; cur < buffer->rlimit; )
902 unsigned char c = *cur++;
906 /* Check for a (trigaph?) backslash escaping the newline. */
907 if (!backslash_start (pfile, cur - 2))
910 handle_newline (cur, buffer->rlimit, c);
916 buffer->cur = cur - 1; /* Leave newline for caller. */
920 /* TAB points to a \t character. Update col_adjust so we track the
923 adjust_column (pfile, tab)
927 /* Zero-based column. */
928 unsigned int col = CPP_BUF_COLUMN (pfile->buffer, tab);
930 /* Round it up to multiple of the tabstop, but subtract 1 since the
931 tab itself occupies a character position. */
932 pfile->col_adjust += (CPP_OPTION (pfile, tabstop)
933 - col % CPP_OPTION (pfile, tabstop)) - 1;
936 /* Skips whitespace, stopping at next non-whitespace character.
937 Adjusts pfile->col_adjust to account for tabs. This enables tokens
938 to be assigned the correct column. */
940 skip_whitespace (pfile, in_directive)
944 cpp_buffer *buffer = pfile->buffer;
945 unsigned short warned = 0;
947 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
948 while (buffer->cur < buffer->rlimit)
950 unsigned char c = *buffer->cur;
956 /* Horizontal space always OK. */
960 adjust_column (pfile, buffer->cur - 1);
961 /* Must be \f \v or \0. */
965 cpp_warning_with_line (pfile, CPP_BUF_LINE (buffer),
966 CPP_BUF_COL (buffer),
967 "embedded null character ignored");
970 else if (in_directive && CPP_PEDANTIC (pfile))
971 cpp_pedwarn_with_line (pfile, CPP_BUF_LINE (buffer),
972 CPP_BUF_COL (buffer),
973 "%s in preprocessing directive",
974 c == '\f' ? "form feed" : "vertical tab");
978 /* Parse (append) an identifier. Calculates the hash value of the
979 token while parsing, for performance. The algorithm *must* match
981 static const U_CHAR *
982 parse_name (pfile, tok, cur, rlimit)
985 const U_CHAR *cur, *rlimit;
995 if (! is_idchar (*cur))
997 /* $ is not a identifier character in the standard, but is
998 commonly accepted as an extension. Don't warn about it in
999 skipped conditional blocks. */
1000 if (*cur == '$' && CPP_PEDANTIC (pfile) && ! pfile->skipping)
1002 CPP_BUFFER (pfile)->cur = cur;
1003 cpp_pedwarn (pfile, "'$' character in identifier");
1006 r = HASHSTEP (r, cur);
1011 if (tok->val.node == 0)
1012 tok->val.node = _cpp_lookup_with_hash (pfile, name, len, r);
1015 unsigned int oldlen = tok->val.node->length;
1016 U_CHAR *newname = alloca (oldlen + len);
1017 memcpy (newname, tok->val.node->name, oldlen);
1018 memcpy (newname + oldlen, name, len);
1019 tok->val.node = cpp_lookup (pfile, newname, len + oldlen);
1025 /* Parse (append) a number. */
1027 parse_number (pfile, list, name)
1032 const unsigned char *name_limit;
1033 unsigned char *namebuf;
1034 cpp_buffer *buffer = pfile->buffer;
1035 register const unsigned char *cur = buffer->cur;
1038 name_limit = list->namebuf + list->name_cap;
1039 namebuf = list->namebuf + list->name_used;
1041 for (; cur < buffer->rlimit && namebuf < name_limit; )
1043 unsigned char c = *namebuf = *cur; /* Copy a single char. */
1045 /* Perhaps we should accept '$' here if we accept it for
1046 identifiers. We know namebuf[-1] is safe, because for c to
1047 be a sign we must have pushed at least one character. */
1048 if (!is_numchar (c) && c != '.' && ! VALID_SIGN (c, namebuf[-1]))
1055 /* Run out of name space? */
1056 if (cur < buffer->rlimit)
1058 list->name_used = namebuf - list->namebuf;
1059 auto_expand_name_space (list);
1065 name->len = namebuf - name->text;
1066 list->name_used = namebuf - list->namebuf;
1069 /* Places a string terminated by an unescaped TERMINATOR into a
1070 cpp_string, which should be expandable and thus at the top of the
1071 list's stack. Handles embedded trigraphs, if necessary, and
1074 Can be used for character constants (terminator = '\''), string
1075 constants ('"') and angled headers ('>'). Multi-line strings are
1076 allowed, except for within directives. */
1079 parse_string (pfile, list, token, terminator)
1083 unsigned int terminator;
1085 cpp_buffer *buffer = pfile->buffer;
1086 cpp_string *name = &token->val.str;
1087 register const unsigned char *cur = buffer->cur;
1088 const unsigned char *name_limit;
1089 unsigned char *namebuf;
1090 unsigned int null_count = 0;
1091 unsigned int trigraphed = list->name_used;
1094 name_limit = list->namebuf + list->name_cap;
1095 namebuf = list->namebuf + list->name_used;
1097 for (; cur < buffer->rlimit && namebuf < name_limit; )
1099 unsigned int c = *namebuf++ = *cur++; /* Copy a single char. */
1103 else if (c == terminator || is_vspace (c))
1105 /* Needed for trigraph_replace and multiline string warning. */
1108 /* Scan for trigraphs before checking if backslash-escaped. */
1109 if ((CPP_OPTION (pfile, trigraphs)
1110 || CPP_OPTION (pfile, warn_trigraphs))
1111 && namebuf - (list->namebuf + trigraphed) >= 3)
1113 namebuf = trigraph_replace (pfile, list->namebuf + trigraphed,
1115 /* The test above guarantees trigraphed will be positive. */
1116 trigraphed = namebuf - list->namebuf - 2;
1119 namebuf--; /* Drop the newline / terminator from the name. */
1122 /* Drop a backslash newline, and continue. */
1123 if (namebuf[-1] == '\\')
1125 handle_newline (cur, buffer->rlimit, c);
1132 /* In assembly language, silently terminate strings of
1133 either variety at end of line. This is a kludge
1134 around not knowing where comments are. */
1135 if (CPP_OPTION (pfile, lang_asm))
1138 /* Character constants and header names may not extend
1139 over multiple lines. In Standard C, neither may
1140 strings. We accept multiline strings as an
1141 extension. (Even in directives - otherwise, glibc's
1142 longlong.h breaks.) */
1143 if (terminator != '"')
1146 cur++; /* Move forwards again. */
1148 if (pfile->multiline_string_line == 0)
1150 pfile->multiline_string_line = token->line;
1151 pfile->multiline_string_column = token->col;
1152 if (CPP_PEDANTIC (pfile))
1153 cpp_pedwarn (pfile, "multi-line string constant");
1157 handle_newline (cur, buffer->rlimit, c);
1161 unsigned char *temp;
1163 /* An odd number of consecutive backslashes represents
1164 an escaped terminator. */
1166 while (temp >= name->text && *temp == '\\')
1169 if ((namebuf - temp) & 1)
1176 /* Run out of name space? */
1177 if (cur < buffer->rlimit)
1179 list->name_used = namebuf - list->namebuf;
1180 auto_expand_name_space (list);
1184 /* We may not have trigraph-replaced the input for this code path,
1185 but as the input is in error by being unterminated we don't
1186 bother. Prevent warnings about no newlines at EOF. */
1187 if (is_vspace (cur[-1]))
1191 cpp_error (pfile, "missing terminating %c character", (int) terminator);
1193 if (terminator == '\"' && pfile->multiline_string_line != list->line
1194 && pfile->multiline_string_line != 0)
1196 cpp_error_with_line (pfile, pfile->multiline_string_line,
1197 pfile->multiline_string_column,
1198 "possible start of unterminated string literal");
1199 pfile->multiline_string_line = 0;
1204 name->len = namebuf - name->text;
1205 list->name_used = namebuf - list->namebuf;
1208 cpp_warning (pfile, (null_count > 1 ? "null characters preserved"
1209 : "null character preserved"));
1212 /* The character TYPE helps us distinguish comment types: '*' = C
1213 style, '/' = C++ style. For code simplicity, the stored comment
1214 includes the comment start and any terminator. */
1216 #define COMMENT_START_LEN 2
1218 save_comment (list, token, from, len, type)
1221 const unsigned char *from;
1225 unsigned char *buffer;
1227 len += COMMENT_START_LEN;
1229 if (list->name_used + len > list->name_cap)
1230 _cpp_expand_name_space (list, len);
1232 INIT_TOKEN_STR (list, token);
1233 token->type = CPP_COMMENT;
1234 token->val.str.len = len;
1236 buffer = list->namebuf + list->name_used;
1237 list->name_used += len;
1239 /* Copy the comment. */
1250 memcpy (buffer, from, len - COMMENT_START_LEN);
1254 * The tokenizer's main loop. Returns a token list, representing a
1255 * logical line in the input file. On EOF after some tokens have
1256 * been processed, we return immediately. Then in next call, or if
1257 * EOF occurred at the beginning of a logical line, a single CPP_EOF
1258 * token is placed in the list.
1260 * Implementation relies almost entirely on lookback, rather than
1261 * looking forwards. This means that tokenization requires just
1262 * a single pass of the file, even in the presence of trigraphs and
1263 * escaped newlines, providing significant performance benefits.
1264 * Trigraph overhead is negligible if they are disabled, and low
1265 * even when enabled.
1268 #define KNOWN_DIRECTIVE() (list->directive != 0)
1269 #define MIGHT_BE_DIRECTIVE() \
1270 (cur_token == &list->tokens[first_token + 1] && cur_token[-1].type == CPP_HASH)
1273 lex_line (pfile, list)
1277 cpp_token *cur_token, *token_limit, *first;
1278 cpp_buffer *buffer = pfile->buffer;
1279 const unsigned char *cur = buffer->cur;
1280 unsigned char flags = 0;
1281 unsigned int first_token = list->tokens_used;
1283 if (!(list->flags & LIST_OFFSET))
1286 list->file = buffer->nominal_fname;
1287 list->line = CPP_BUF_LINE (buffer);
1288 pfile->col_adjust = 0;
1289 pfile->in_lex_line = 1;
1290 if (cur == buffer->buf)
1291 list->flags |= BEG_OF_FILE;
1294 token_limit = list->tokens + list->tokens_cap;
1295 cur_token = list->tokens + list->tokens_used;
1297 for (; cur < buffer->rlimit && cur_token < token_limit;)
1301 /* Optimize non-vertical whitespace skipping; most tokens are
1302 probably separated by whitespace. (' ' '\t' '\v' '\f' '\0'). */
1307 skip_whitespace (pfile, (list->tokens[first_token].type == CPP_HASH
1308 && cur_token > &list->tokens[first_token]));
1312 if (cur == buffer->rlimit)
1318 /* Initialize current token. CPP_EOF will not be fixed up by
1319 expand_name_space. */
1320 list->tokens_used = cur_token - list->tokens + 1;
1321 cur_token->type = CPP_EOF;
1322 cur_token->col = CPP_BUF_COLUMN (buffer, cur);
1323 cur_token->line = CPP_BUF_LINE (buffer);
1324 cur_token->flags = flags;
1329 case '0': case '1': case '2': case '3': case '4':
1330 case '5': case '6': case '7': case '8': case '9':
1334 cur--; /* Backup character. */
1335 prev_dot = PREV_TOKEN_TYPE == CPP_DOT && IMMED_TOKEN ();
1338 INIT_TOKEN_STR (list, cur_token);
1339 /* Prepend an immediately previous CPP_DOT token. */
1342 if (list->name_cap == list->name_used)
1343 auto_expand_name_space (list);
1345 cur_token->val.str.len = 1;
1346 list->namebuf[list->name_used++] = '.';
1350 cur_token->type = CPP_NUMBER; /* Before parse_number. */
1352 parse_number (pfile, list, &cur_token->val.str);
1355 /* Check for # 123 form of #line. */
1356 if (MIGHT_BE_DIRECTIVE ())
1357 list->directive = _cpp_check_linemarker (pfile, cur_token,
1358 !(cur_token[-1].flags
1365 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1366 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1367 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1368 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1370 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1371 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1372 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1373 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1375 cur--; /* Backup character. */
1376 cur_token->val.node = 0;
1377 cur_token->type = CPP_NAME; /* Identifier, macro etc. */
1380 cur = parse_name (pfile, cur_token, cur, buffer->rlimit);
1382 if (MIGHT_BE_DIRECTIVE ())
1383 list->directive = _cpp_check_directive (pfile, cur_token,
1384 !(list->tokens[0].flags
1386 /* Convert named operators to their proper types. */
1387 if (cur_token->val.node->type == T_OPERATOR)
1389 cur_token->flags |= NAMED_OP;
1390 cur_token->type = cur_token->val.node->value.code;
1398 cur_token->type = c == '\'' ? CPP_CHAR : CPP_STRING;
1399 /* Do we have a wide string? */
1400 if (cur_token[-1].type == CPP_NAME && IMMED_TOKEN ()
1401 && cur_token[-1].val.node == pfile->spec_nodes->n_L)
1402 BACKUP_TOKEN (c == '\'' ? CPP_WCHAR : CPP_WSTRING);
1405 /* Here c is one of ' " or >. */
1406 INIT_TOKEN_STR (list, cur_token);
1408 parse_string (pfile, list, cur_token, c);
1414 cur_token->type = CPP_DIV;
1417 if (PREV_TOKEN_TYPE == CPP_DIV)
1419 /* We silently allow C++ comments in system headers,
1420 irrespective of conformance mode, because lots of
1421 broken systems do that and trying to clean it up
1422 in fixincludes is a nightmare. */
1423 if (CPP_IN_SYSTEM_HEADER (pfile))
1424 goto do_line_comment;
1425 else if (CPP_OPTION (pfile, cplusplus_comments))
1427 if (CPP_OPTION (pfile, c89) && CPP_PEDANTIC (pfile)
1428 && ! buffer->warned_cplusplus_comments)
1432 "C++ style comments are not allowed in ISO C89");
1434 "(this will be reported only once per input file)");
1435 buffer->warned_cplusplus_comments = 1;
1439 #if 0 /* Leave until new lexer in place. */
1442 "comment start split across lines");
1444 if (skip_line_comment (pfile))
1445 cpp_warning (pfile, "multi-line comment");
1447 /* Back-up to first '-' or '/'. */
1449 if (!CPP_OPTION (pfile, discard_comments)
1450 && (!KNOWN_DIRECTIVE()
1451 || (list->directive->flags & COMMENTS)))
1452 save_comment (list, cur_token++, cur,
1453 buffer->cur - cur, c);
1466 cur_token->type = CPP_MULT;
1469 if (PREV_TOKEN_TYPE == CPP_DIV)
1472 #if 0 /* Leave until new lexer in place. */
1475 "comment start '/*' split across lines");
1477 if (skip_block_comment (pfile))
1478 cpp_error_with_line (pfile, list->line, cur_token[-1].col,
1479 "unterminated comment");
1480 #if 0 /* Leave until new lexer in place. */
1481 else if (buffer->cur[-2] != '*')
1483 "comment end '*/' split across lines");
1485 /* Back up to opening '/'. */
1487 if (!CPP_OPTION (pfile, discard_comments)
1488 && (!KNOWN_DIRECTIVE()
1489 || (list->directive->flags & COMMENTS)))
1490 save_comment (list, cur_token++, cur,
1491 buffer->cur - cur, c);
1498 else if (CPP_OPTION (pfile, cplusplus))
1500 /* In C++, there are .* and ->* operators. */
1501 if (PREV_TOKEN_TYPE == CPP_DEREF)
1502 BACKUP_TOKEN (CPP_DEREF_STAR);
1503 else if (PREV_TOKEN_TYPE == CPP_DOT)
1504 BACKUP_TOKEN (CPP_DOT_STAR);
1512 handle_newline (cur, buffer->rlimit, c);
1513 if (PREV_TOKEN_TYPE == CPP_BACKSLASH)
1517 /* Remove the escaped newline. Then continue to process
1518 any interrupted name or number. */
1520 /* Backslash-newline may not be immediately followed by
1521 EOF (C99 5.1.1.2). */
1522 if (cur >= buffer->rlimit)
1524 cpp_pedwarn (pfile, "backslash-newline at end of file");
1530 if (cur_token->type == CPP_NAME)
1532 else if (cur_token->type == CPP_NUMBER)
1533 goto continue_number;
1536 /* Remember whitespace setting. */
1537 flags = cur_token->flags;
1544 "backslash and newline separated by space");
1547 else if (MIGHT_BE_DIRECTIVE ())
1549 /* "Null directive." C99 6.10.7: A preprocessing
1550 directive of the form # <new-line> has no effect.
1552 But it is still a directive, and therefore disappears
1555 if (cur_token->flags & PREV_WHITE
1556 && CPP_WTRADITIONAL (pfile))
1557 cpp_warning (pfile, "K+R C ignores #\\n with the # indented");
1560 /* Skip vertical space until we have at least one token to
1562 if (cur_token != &list->tokens[first_token])
1564 list->line = CPP_BUF_LINE (buffer);
1568 if (IMMED_TOKEN () && PREV_TOKEN_TYPE == CPP_MINUS)
1569 REVISE_TOKEN (CPP_MINUS_MINUS);
1571 PUSH_TOKEN (CPP_MINUS);
1576 /* The digraph flag checking ensures that ## and %:%:
1577 are interpreted as CPP_PASTE, but #%: and %:# are not. */
1578 if (PREV_TOKEN_TYPE == CPP_HASH && IMMED_TOKEN ()
1579 && ((cur_token->flags ^ cur_token[-1].flags) & DIGRAPH) == 0)
1580 REVISE_TOKEN (CPP_PASTE);
1582 PUSH_TOKEN (CPP_HASH);
1586 cur_token->type = CPP_COLON;
1589 if (PREV_TOKEN_TYPE == CPP_COLON
1590 && CPP_OPTION (pfile, cplusplus))
1591 BACKUP_TOKEN (CPP_SCOPE);
1592 else if (CPP_OPTION (pfile, digraphs))
1594 /* Digraph: "<:" is a '[' */
1595 if (PREV_TOKEN_TYPE == CPP_LESS)
1596 BACKUP_DIGRAPH (CPP_OPEN_SQUARE);
1597 /* Digraph: "%:" is a '#' */
1598 else if (PREV_TOKEN_TYPE == CPP_MOD)
1600 (--cur_token)->flags |= DIGRAPH;
1609 if (IMMED_TOKEN () && PREV_TOKEN_TYPE == CPP_AND)
1610 REVISE_TOKEN (CPP_AND_AND);
1612 PUSH_TOKEN (CPP_AND);
1617 if (IMMED_TOKEN () && PREV_TOKEN_TYPE == CPP_OR)
1618 REVISE_TOKEN (CPP_OR_OR);
1620 PUSH_TOKEN (CPP_OR);
1624 if (IMMED_TOKEN () && PREV_TOKEN_TYPE == CPP_PLUS)
1625 REVISE_TOKEN (CPP_PLUS_PLUS);
1627 PUSH_TOKEN (CPP_PLUS);
1631 /* This relies on equidistance of "?=" and "?" tokens. */
1632 if (IMMED_TOKEN () && PREV_TOKEN_TYPE <= CPP_LAST_EQ)
1633 REVISE_TOKEN (PREV_TOKEN_TYPE + (CPP_EQ_EQ - CPP_EQ));
1635 PUSH_TOKEN (CPP_EQ);
1639 cur_token->type = CPP_GREATER;
1642 if (PREV_TOKEN_TYPE == CPP_GREATER)
1643 BACKUP_TOKEN (CPP_RSHIFT);
1644 else if (PREV_TOKEN_TYPE == CPP_MINUS)
1645 BACKUP_TOKEN (CPP_DEREF);
1646 else if (CPP_OPTION (pfile, digraphs))
1648 /* Digraph: ":>" is a ']' */
1649 if (PREV_TOKEN_TYPE == CPP_COLON)
1650 BACKUP_DIGRAPH (CPP_CLOSE_SQUARE);
1651 /* Digraph: "%>" is a '}' */
1652 else if (PREV_TOKEN_TYPE == CPP_MOD)
1653 BACKUP_DIGRAPH (CPP_CLOSE_BRACE);
1660 if (IMMED_TOKEN () && PREV_TOKEN_TYPE == CPP_LESS)
1662 REVISE_TOKEN (CPP_LSHIFT);
1665 /* Is this the beginning of a header name? */
1666 if (KNOWN_DIRECTIVE () && (list->directive->flags & INCL))
1668 c = '>'; /* Terminator. */
1669 cur_token->type = CPP_HEADER_NAME;
1670 goto do_parse_string;
1672 PUSH_TOKEN (CPP_LESS);
1676 /* Digraph: "<%" is a '{' */
1677 cur_token->type = CPP_MOD;
1678 if (IMMED_TOKEN () && PREV_TOKEN_TYPE == CPP_LESS
1679 && CPP_OPTION (pfile, digraphs))
1680 BACKUP_DIGRAPH (CPP_OPEN_BRACE);
1685 if (cur + 1 < buffer->rlimit && *cur == '?'
1686 && trigraph_map[cur[1]] && trigraph_ok (pfile, cur + 1))
1688 /* Handle trigraph. */
1692 case '(': goto make_open_square;
1693 case ')': goto make_close_square;
1694 case '<': goto make_open_brace;
1695 case '>': goto make_close_brace;
1696 case '=': goto make_hash;
1697 case '!': goto make_or;
1698 case '-': goto make_complement;
1699 case '/': goto make_backslash;
1700 case '\'': goto make_xor;
1703 if (IMMED_TOKEN () && CPP_OPTION (pfile, cplusplus))
1705 /* GNU C++ defines <? and >? operators. */
1706 if (PREV_TOKEN_TYPE == CPP_LESS)
1708 REVISE_TOKEN (CPP_MIN);
1711 else if (PREV_TOKEN_TYPE == CPP_GREATER)
1713 REVISE_TOKEN (CPP_MAX);
1717 PUSH_TOKEN (CPP_QUERY);
1721 if (PREV_TOKEN_TYPE == CPP_DOT && cur_token[-2].type == CPP_DOT
1723 && !(cur_token[-1].flags & PREV_WHITE))
1726 PUSH_TOKEN (CPP_ELLIPSIS);
1729 PUSH_TOKEN (CPP_DOT);
1733 case '~': PUSH_TOKEN (CPP_COMPL); break;
1735 case '^': PUSH_TOKEN (CPP_XOR); break;
1737 case '{': PUSH_TOKEN (CPP_OPEN_BRACE); break;
1739 case '}': PUSH_TOKEN (CPP_CLOSE_BRACE); break;
1741 case '[': PUSH_TOKEN (CPP_OPEN_SQUARE); break;
1743 case ']': PUSH_TOKEN (CPP_CLOSE_SQUARE); break;
1745 case '\\': PUSH_TOKEN (CPP_BACKSLASH); break;
1746 case '!': PUSH_TOKEN (CPP_NOT); break;
1747 case ',': PUSH_TOKEN (CPP_COMMA); break;
1748 case ';': PUSH_TOKEN (CPP_SEMICOLON); break;
1749 case '(': PUSH_TOKEN (CPP_OPEN_PAREN); break;
1750 case ')': PUSH_TOKEN (CPP_CLOSE_PAREN); break;
1753 if (CPP_OPTION (pfile, dollars_in_ident))
1757 cur_token->val.aux = c;
1758 PUSH_TOKEN (CPP_OTHER);
1763 /* Run out of token space? */
1764 if (cur_token == token_limit)
1766 list->tokens_used = cur_token - list->tokens;
1767 _cpp_expand_token_space (list, 256);
1771 cur_token->flags = flags;
1772 if (cur_token == &list->tokens[first_token] && pfile->done_initializing)
1774 if (cur > buffer->buf && !is_vspace (cur[-1]))
1775 cpp_pedwarn_with_line (pfile, CPP_BUF_LINE (buffer),
1776 CPP_BUF_COLUMN (buffer, cur),
1777 "no newline at end of file");
1778 cur_token++->type = CPP_EOF;
1782 /* All tokens are allocated, so the memory location is fixed. */
1783 first = &list->tokens[first_token];
1785 /* Don't complain about the null directive, nor directives in
1786 assembly source: we don't know where the comments are, and # may
1787 introduce assembler pseudo-ops. Don't complain about invalid
1788 directives in skipped conditional groups (6.10 p4). */
1789 if (first->type == CPP_HASH && list->directive == 0 && !pfile->skipping
1790 && cur_token > first + 1 && !CPP_OPTION (pfile, lang_asm))
1792 if (first[1].type == CPP_NAME)
1793 cpp_error (pfile, "invalid preprocessing directive #%s",
1794 first[1].val.node->name);
1796 cpp_error (pfile, "invalid preprocessing directive");
1799 /* Put EOF at end of known directives. This covers "directives do
1800 not extend beyond the end of the line (description 6.10 part 2)". */
1801 if (KNOWN_DIRECTIVE () || !pfile->done_initializing)
1803 pfile->first_directive_token = first;
1804 cur_token++->type = CPP_EOF;
1807 first->flags |= BOL;
1808 if (first_token != 0)
1809 /* 6.10.3.10: Within the sequence of preprocessing tokens making
1810 up the invocation of a function-like macro, new line is
1811 considered a normal white-space character. */
1812 first->flags |= PREV_WHITE;
1815 list->tokens_used = cur_token - list->tokens;
1816 pfile->in_lex_line = 0;
1819 /* Write the spelling of a token TOKEN, with any appropriate
1820 whitespace before it, to FP. PREV is the previous token, which
1821 is used to determine if we need to shove in an extra space in order
1822 to avoid accidental token paste. If WHITE is 0, do not insert any
1823 leading whitespace. */
1825 output_token (pfile, fp, token, prev, white)
1828 const cpp_token *token, *prev;
1835 if (token->col && (token->flags & BOL))
1837 /* Supply enough whitespace to put this token in its original
1838 column. Don't bother trying to reconstruct tabs; we can't
1839 get it right in general, and nothing ought to care. (Yes,
1840 some things do care; the fault lies with them.) */
1841 unsigned int spaces = token->col - 1;
1846 else if (token->flags & PREV_WHITE)
1849 /* Check for and prevent accidental token pasting.
1850 In addition to the cases handled by can_paste, consider
1852 a + ++b - if there is not a space between the + and ++, it
1853 will be misparsed as a++ + b. But + ## ++ doesn't produce
1856 && (can_paste (pfile, prev, token, &dummy) != CPP_EOF
1857 || (prev->type == CPP_PLUS && token->type == CPP_PLUS_PLUS)
1858 || (prev->type == CPP_MINUS && token->type == CPP_MINUS_MINUS)))
1862 switch (TOKEN_SPELL (token))
1864 case SPELL_OPERATOR:
1866 const unsigned char *spelling;
1868 if (token->flags & DIGRAPH)
1869 spelling = digraph_spellings[token->type - CPP_FIRST_DIGRAPH];
1870 else if (token->flags & NAMED_OP)
1873 spelling = TOKEN_NAME (token);
1875 ufputs (spelling, fp);
1881 ufputs (token->val.node->name, fp);
1886 if (token->type == CPP_WSTRING || token->type == CPP_WCHAR)
1889 if (token->type == CPP_STRING || token->type == CPP_WSTRING)
1891 if (token->type == CPP_CHAR || token->type == CPP_WCHAR)
1894 fwrite (token->val.str.text, 1, token->val.str.len, fp);
1896 if (token->type == CPP_STRING || token->type == CPP_WSTRING)
1898 if (token->type == CPP_CHAR || token->type == CPP_WCHAR)
1904 putc (token->val.aux, fp);
1908 /* Placemarker or EOF - no output. (Macro args are handled
1914 /* Dump the original user's spelling of argument index ARG_NO to the
1915 macro whose expansion is LIST. */
1917 dump_param_spelling (fp, list, arg_no)
1919 const cpp_toklist *list;
1920 unsigned int arg_no;
1922 const U_CHAR *param = list->namebuf;
1925 param += ustrlen (param) + 1;
1929 /* Output all the tokens of LIST, starting at TOKEN, to FP. */
1931 cpp_output_list (pfile, fp, list, token)
1934 const cpp_toklist *list;
1935 const cpp_token *token;
1937 const cpp_token *limit = list->tokens + list->tokens_used;
1938 const cpp_token *prev = 0;
1941 while (token < limit)
1943 /* XXX Find some way we can write macro args from inside
1944 output_token/spell_token. */
1945 if (token->type == CPP_MACRO_ARG)
1947 if (white && token->flags & PREV_WHITE)
1949 if (token->flags & STRINGIFY_ARG)
1951 dump_param_spelling (fp, list, token->val.aux);
1954 output_token (pfile, fp, token, prev, white);
1955 if (token->flags & PASTE_LEFT)
1964 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
1965 already contain the enough space to hold the token's spelling.
1966 Returns a pointer to the character after the last character
1969 static unsigned char *
1970 spell_token (pfile, token, buffer)
1971 cpp_reader *pfile; /* Would be nice to be rid of this... */
1972 const cpp_token *token;
1973 unsigned char *buffer;
1975 switch (TOKEN_SPELL (token))
1977 case SPELL_OPERATOR:
1979 const unsigned char *spelling;
1982 if (token->flags & DIGRAPH)
1983 spelling = digraph_spellings[token->type - CPP_FIRST_DIGRAPH];
1984 else if (token->flags & NAMED_OP)
1987 spelling = TOKEN_NAME (token);
1989 while ((c = *spelling++) != '\0')
1996 memcpy (buffer, token->val.node->name, token->val.node->length);
1997 buffer += token->val.node->length;
2002 if (token->type == CPP_WSTRING || token->type == CPP_WCHAR)
2005 if (token->type == CPP_STRING || token->type == CPP_WSTRING)
2007 if (token->type == CPP_CHAR || token->type == CPP_WCHAR)
2010 memcpy (buffer, token->val.str.text, token->val.str.len);
2011 buffer += token->val.str.len;
2013 if (token->type == CPP_STRING || token->type == CPP_WSTRING)
2015 if (token->type == CPP_CHAR || token->type == CPP_WCHAR)
2021 *buffer++ = token->val.aux;
2025 cpp_ice (pfile, "Unspellable token %s", TOKEN_NAME (token));
2032 /* Macro expansion algorithm.
2034 Macro expansion is implemented by a single-pass algorithm; there are
2035 no rescan passes involved. cpp_get_token expands just enough to be
2036 able to return a token to the caller, a consequence is that when it
2037 returns the preprocessor can be in a state of mid-expansion. The
2038 algorithm does not work by fully expanding a macro invocation into
2039 some kind of token list, and then returning them one by one.
2041 Our expansion state is recorded in a context stack. We start out with
2042 a single context on the stack, let's call it base context. This
2043 consists of the token list returned by lex_line that forms the next
2044 logical line in the source file.
2046 The current level in the context stack is stored in the cur_context
2047 member of the cpp_reader structure. The context it references keeps,
2048 amongst other things, a count of how many tokens form that context and
2049 our position within those tokens.
2051 Fundamentally, calling cpp_get_token will return the next token from
2052 the current context. If we're at the end of the current context, that
2053 context is popped from the stack first, unless it is the base context,
2054 in which case the next logical line is lexed from the source file.
2056 However, before returning the token, if it is a CPP_NAME token
2057 _cpp_get_token checks to see if it is a macro and if it is enabled.
2058 Each time it encounters a macro name, it calls push_macro_context.
2059 This function checks that the macro should be expanded (with
2060 is_macro_enabled), and if so pushes a new macro context on the stack
2061 which becomes the current context. It then loops back to read the
2062 first token of the macro context.
2064 A macro context basically consists of the token list representing the
2065 macro's replacement list, which was saved in the hash table by
2066 save_macro_expansion when its #define statement was parsed. If the
2067 macro is function-like, it also contains the tokens that form the
2068 arguments to the macro. I say more about macro arguments below, but
2069 for now just saying that each argument is a set of pointers to tokens
2072 When taking tokens from a macro context, we may get a CPP_MACRO_ARG
2073 token. This represents an argument passed to the macro, with the
2074 argument number stored in the token's AUX field. The argument should
2075 be substituted, this is achieved by pushing an "argument context". An
2076 argument context is just refers to the tokens forming the argument,
2077 which are obtained directly from the macro context. The STRINGIFY
2078 flag on a CPP_MACRO_ARG token indicates that the argument should be
2081 Here's a few simple rules the context stack obeys:-
2083 1) The lex_line token list is always context zero.
2085 2) Context 1, if it exists, must be a macro context.
2087 3) An argument context can only appear above a macro context.
2089 4) A macro context can appear above the base context, another macro
2090 context, or an argument context.
2092 5) These imply that the minimal level of an argument context is 2.
2094 The only tricky thing left is ensuring that macros are enabled and
2095 disabled correctly. The algorithm controls macro expansion by the
2096 level of the context a token is taken from in the context stack. If a
2097 token is taken from a level equal to no_expand_level (a member of
2098 struct cpp_reader), no expansion is performed.
2100 When popping a context off the stack, if no_expand_level equals the
2101 level of the popped context, it is reduced by one to match the new
2102 context level, so that expansion is still disabled. It does not
2103 increase if a context is pushed, though. It starts out life as
2104 UINT_MAX, which has the effect that initially macro expansion is
2105 enabled. I explain how this mechanism works below.
2107 The standard requires:-
2109 1) Arguments to be fully expanded before substitution.
2111 2) Stringified arguments to not be expanded, nor the tokens
2112 immediately surrounding a ## operator.
2114 3) Continual rescanning until there are no more macros left to
2117 4) Once a macro has been expanded in stage 1) or 3), it cannot be
2118 expanded again during later rescans. This prevents infinite
2121 The first thing to observe is that stage 3) is mostly redundant.
2122 Since a macro is disabled once it has been expanded, how can a rescan
2123 find an unexpanded macro name? There are only two cases where this is
2126 a) If the macro name results from a token paste operation.
2128 b) If the macro in question is a function-like macro that hasn't
2129 already been expanded because previously there was not the required
2130 '(' token immediately following it. This is only possible when an
2131 argument is substituted, and after substitution the last token of
2132 the argument can bind with a parenthesis appearing in the tokens
2133 following the substitution. Note that if the '(' appears within the
2134 argument, the ')' must too, as expanding macro arguments cannot
2135 "suck in" tokens outside the argument.
2137 So we tackle this as follows. When parsing the macro invocation for
2138 arguments, we record the tokens forming each argument as a list of
2139 pointers to those tokens. We do not expand any tokens that are "raw",
2140 i.e. directly from the macro invocation, but other tokens that come
2141 from (nested) argument substitution are fully expanded.
2143 This is achieved by setting the no_expand_level to that of the macro
2144 invocation. A CPP_MACRO_ARG token never appears in the list of tokens
2145 forming an argument, because parse_args (indirectly) calls
2146 get_raw_token which automatically pushes argument contexts and traces
2147 into them. Since these contexts are at a higher level than the
2148 no_expand_level, they get fully macro expanded.
2150 "Raw" and non-raw tokens are separated in arguments by null pointers,
2151 with the policy that the initial state of an argument is raw. If the
2152 first token is not raw, it should be preceded by a null pointer. When
2153 tracing through the tokens of an argument context, each time
2154 get_raw_token encounters a null pointer, it toggles the flag
2157 This flag, when set, indicates to is_macro_disabled that we are
2158 reading raw tokens which should be macro-expanded. Similarly, if
2159 clear, is_macro_disabled suppresses re-expansion.
2161 It's probably time for an example.
2165 #define xstr(y) str(y hash)
2167 xstr(hash) // "# hash"
2169 In the invocation of str, parse_args turns off macro expansion and so
2170 parses the argument as <hash>. This is the only token (pointer)
2171 passed as the argument to str. Since <hash> is raw there is no need
2172 for an initial null pointer. stringify_arg is called from
2173 get_raw_token when tracing through the expansion of str, since the
2174 argument has the STRINGIFY flag set. stringify_arg turns off
2175 macro_expansion by setting the no_expand_level to that of the argument
2176 context. Thus it gets the token <hash> and stringifies it to "hash"
2179 Similary xstr is passed <hash>. However, when parse_args is parsing
2180 the invocation of str() in xstr's expansion, get_raw_token encounters
2181 a CPP_MACRO_ARG token for y. Transparently to parse_args, it pushes
2182 an argument context, and enters the tokens of the argument,
2183 i.e. <hash>. This is at a higher context level than parse_args
2184 disabled, and so is_macro_disabled permits expansion of it and a macro
2185 context is pushed on top of the argument context. This contains the
2186 <#> token, and the end result is that <hash> is macro expanded.
2187 However, after popping off the argument context, the <hash> of xstr's
2188 expansion does not get macro expanded because we're back at the
2189 no_expand_level. The end result is that the argument passed to str is
2190 <NULL> <#> <NULL> <hash>. Note the nulls - policy is we start off
2191 raw, <#> is not raw, but then <hash> is.
2196 /* Free the storage allocated for macro arguments. */
2198 free_macro_args (args)
2202 free ((PTR) args->tokens);
2207 /* Determines if a macro has been already used (and is therefore
2210 is_macro_disabled (pfile, expansion, token)
2212 const cpp_toklist *expansion;
2213 const cpp_token *token;
2215 cpp_context *context = CURRENT_CONTEXT (pfile);
2217 /* Don't expand anything if this file has already been preprocessed. */
2218 if (CPP_OPTION (pfile, preprocessed))
2221 /* Arguments on either side of ## are inserted in place without
2222 macro expansion (6.10.3.3.2). Conceptually, any macro expansion
2223 occurs during a later rescan pass. The effect is that we expand
2224 iff we would as part of the macro's expansion list, so we should
2225 drop to the macro's context. */
2226 if (IS_ARG_CONTEXT (context))
2228 if (token->flags & PASTED)
2230 else if (!(context->flags & CONTEXT_RAW))
2232 else if (context->flags & (CONTEXT_PASTEL | CONTEXT_PASTER))
2236 /* Have we already used this macro? */
2237 while (context->level > 0)
2239 if (!IS_ARG_CONTEXT (context) && context->u.list == expansion)
2241 /* Raw argument tokens are judged based on the token list they
2243 if (context->flags & CONTEXT_RAW)
2244 context = pfile->contexts + context->level;
2249 /* Function-like macros may be disabled if the '(' is not in the
2250 current context. We check this without disrupting the context
2252 if (expansion->paramc >= 0)
2254 const cpp_token *next;
2255 unsigned int prev_nme;
2257 context = CURRENT_CONTEXT (pfile);
2258 /* Drop down any contexts we're at the end of: the '(' may
2259 appear in lower macro expansions, or in the rest of the file. */
2260 while (context->posn == context->count && context > pfile->contexts)
2263 /* If we matched, we are disabled, as we appear in the
2264 expansion of each macro we meet. */
2265 if (!IS_ARG_CONTEXT (context) && context->u.list == expansion)
2269 prev_nme = pfile->no_expand_level;
2270 pfile->no_expand_level = context - pfile->contexts;
2271 next = _cpp_get_token (pfile);
2272 restore_macro_expansion (pfile, prev_nme);
2273 if (next->type != CPP_OPEN_PAREN)
2275 _cpp_push_token (pfile, next);
2276 if (CPP_WTRADITIONAL (pfile))
2278 "function macro %s must be used with arguments in traditional C",
2279 token->val.node->name);
2287 /* Add a token to the set of tokens forming the arguments to the macro
2288 being parsed in parse_args. */
2290 save_token (args, token)
2292 const cpp_token *token;
2294 if (args->used == args->capacity)
2296 args->capacity += args->capacity + 100;
2297 args->tokens = (const cpp_token **)
2298 xrealloc ((PTR) args->tokens,
2299 args->capacity * sizeof (const cpp_token *));
2301 args->tokens[args->used++] = token;
2304 /* Take and save raw tokens until we finish one argument. Empty
2305 arguments are saved as a single CPP_PLACEMARKER token. */
2306 static const cpp_token *
2307 parse_arg (pfile, var_args, paren_context, args, pcount)
2310 unsigned int paren_context;
2312 unsigned int *pcount;
2314 const cpp_token *token;
2315 unsigned int paren = 0, count = 0;
2316 int raw, was_raw = 1;
2318 for (count = 0;; count++)
2320 token = _cpp_get_token (pfile);
2322 switch (token->type)
2327 case CPP_OPEN_PAREN:
2331 case CPP_CLOSE_PAREN:
2337 /* Commas are not terminators within parantheses or var_args. */
2338 if (paren || var_args)
2342 case CPP_EOF: /* Error reported by caller. */
2346 raw = pfile->cur_context <= paren_context;
2350 save_token (args, 0);
2353 save_token (args, token);
2359 /* Duplicate the placemarker. Then we can set its flags and
2360 position and safely be using more than one. */
2361 save_token (args, duplicate_token (pfile, &placemarker_token));
2369 /* This macro returns true if the argument starting at offset O of arglist
2370 A is empty - that is, it's either a single PLACEMARKER token, or a null
2371 pointer followed by a PLACEMARKER. */
2373 #define empty_argument(A, O) \
2374 ((A)->tokens[O] ? (A)->tokens[O]->type == CPP_PLACEMARKER \
2375 : (A)->tokens[(O)+1]->type == CPP_PLACEMARKER)
2377 /* Parse the arguments making up a macro invocation. Nested arguments
2378 are automatically macro expanded, but immediate macros are not
2379 expanded; this enables e.g. operator # to work correctly. Returns
2380 non-zero on error. */
2382 parse_args (pfile, hp, args)
2387 const cpp_token *token;
2388 const cpp_toklist *macro;
2389 unsigned int total = 0;
2390 unsigned int paren_context = pfile->cur_context;
2393 macro = hp->value.expansion;
2398 token = parse_arg (pfile, (argc + 1 == macro->paramc
2399 && (macro->flags & VAR_ARGS)),
2400 paren_context, args, &count);
2401 if (argc < macro->paramc)
2404 args->ends[argc] = total;
2408 while (token->type != CPP_CLOSE_PAREN && token->type != CPP_EOF);
2410 if (token->type == CPP_EOF)
2412 cpp_error(pfile, "unterminated argument list for macro \"%s\"", hp->name);
2415 else if (argc < macro->paramc)
2417 /* A rest argument is allowed to not appear in the invocation at all.
2418 e.g. #define debug(format, args...) ...
2420 This is exactly the same as if the rest argument had received no
2421 tokens - debug("string",); This extension is deprecated. */
2423 if (argc + 1 == macro->paramc && (macro->flags & VAR_ARGS))
2425 /* Duplicate the placemarker. Then we can set its flags and
2426 position and safely be using more than one. */
2427 cpp_token *pm = duplicate_token (pfile, &placemarker_token);
2428 pm->flags = VOID_REST;
2429 save_token (args, pm);
2430 args->ends[argc] = total + 1;
2432 if (CPP_OPTION (pfile, c99) && CPP_PEDANTIC (pfile))
2433 cpp_pedwarn (pfile, "ISO C99 requires rest arguments to be used");
2439 cpp_error (pfile, "not enough arguments for macro \"%s\"", hp->name);
2443 /* An empty argument to an empty function-like macro is fine. */
2444 else if (argc > macro->paramc
2445 && !(macro->paramc == 0 && argc == 1 && empty_argument (args, 0)))
2447 cpp_error (pfile, "too many arguments for macro \"%s\"", hp->name);
2454 /* Adds backslashes before all backslashes and double quotes appearing
2455 in strings. Non-printable characters are converted to octal. */
2457 quote_string (dest, src, len)
2466 if (c == '\\' || c == '"')
2477 sprintf ((char *) dest, "\\%03o", c);
2486 /* Allocates a buffer to hold a token's TEXT, and converts TOKEN to a
2487 CPP_STRING token containing TEXT in quoted form. */
2489 make_string_token (token, text, len)
2496 buf = (U_CHAR *) xmalloc (len * 4);
2497 token->type = CPP_STRING;
2499 token->val.str.text = buf;
2500 token->val.str.len = quote_string (buf, text, len) - buf;
2504 /* Allocates and converts a temporary token to a CPP_NUMBER token,
2505 evaluating to NUMBER. */
2507 alloc_number_token (pfile, number)
2514 result = get_temp_token (pfile);
2516 sprintf (buf, "%d", number);
2518 result->type = CPP_NUMBER;
2520 result->val.str.text = (U_CHAR *) buf;
2521 result->val.str.len = strlen (buf);
2525 /* Returns a temporary token from the temporary token store of PFILE. */
2527 get_temp_token (pfile)
2530 if (pfile->temp_used == pfile->temp_alloced)
2532 if (pfile->temp_used == pfile->temp_cap)
2534 pfile->temp_cap += pfile->temp_cap + 20;
2535 pfile->temp_tokens = (cpp_token **) xrealloc
2536 (pfile->temp_tokens, pfile->temp_cap * sizeof (cpp_token *));
2538 pfile->temp_tokens[pfile->temp_alloced++] = (cpp_token *) xmalloc
2539 (sizeof (cpp_token));
2542 return pfile->temp_tokens[pfile->temp_used++];
2545 /* Release (not free) for re-use the temporary tokens of PFILE. */
2547 release_temp_tokens (pfile)
2550 while (pfile->temp_used)
2552 cpp_token *token = pfile->temp_tokens[--pfile->temp_used];
2554 if (TOKEN_SPELL (token) == SPELL_STRING)
2556 free ((char *) token->val.str.text);
2557 token->val.str.text = 0;
2562 /* Free all of PFILE's dynamically-allocated temporary tokens. */
2564 _cpp_free_temp_tokens (pfile)
2567 if (pfile->temp_tokens)
2569 /* It is possible, though unlikely (looking for '(' of a funlike
2570 macro into EOF), that we haven't released the tokens yet. */
2571 release_temp_tokens (pfile);
2572 while (pfile->temp_alloced)
2573 free (pfile->temp_tokens[--pfile->temp_alloced]);
2574 free (pfile->temp_tokens);
2579 free ((char *) pfile->date->val.str.text);
2581 free ((char *) pfile->time->val.str.text);
2586 /* Copy TOKEN into a temporary token from PFILE's store. */
2588 duplicate_token (pfile, token)
2590 const cpp_token *token;
2592 cpp_token *result = get_temp_token (pfile);
2595 if (TOKEN_SPELL (token) == SPELL_STRING)
2597 U_CHAR *buff = (U_CHAR *) xmalloc (token->val.str.len);
2598 memcpy (buff, token->val.str.text, token->val.str.len);
2599 result->val.str.text = buff;
2604 /* Determine whether two tokens can be pasted together, and if so,
2605 what the resulting token is. Returns CPP_EOF if the tokens cannot
2606 be pasted, or the appropriate type for the merged token if they
2608 static enum cpp_ttype
2609 can_paste (pfile, token1, token2, digraph)
2611 const cpp_token *token1, *token2;
2614 enum cpp_ttype a = token1->type, b = token2->type;
2615 int cxx = CPP_OPTION (pfile, cplusplus);
2617 /* Treat named operators as if they were ordinary NAMEs. */
2618 if (token1->flags & NAMED_OP)
2620 if (token2->flags & NAMED_OP)
2623 if (a <= CPP_LAST_EQ && b == CPP_EQ)
2624 return a + (CPP_EQ_EQ - CPP_EQ);
2629 if (b == a) return CPP_RSHIFT;
2630 if (b == CPP_QUERY && cxx) return CPP_MAX;
2631 if (b == CPP_GREATER_EQ) return CPP_RSHIFT_EQ;
2634 if (b == a) return CPP_LSHIFT;
2635 if (b == CPP_QUERY && cxx) return CPP_MIN;
2636 if (b == CPP_LESS_EQ) return CPP_LSHIFT_EQ;
2637 if (CPP_OPTION (pfile, digraphs))
2640 {*digraph = 1; return CPP_OPEN_SQUARE;} /* <: digraph */
2642 {*digraph = 1; return CPP_OPEN_BRACE;} /* <% digraph */
2646 case CPP_PLUS: if (b == a) return CPP_PLUS_PLUS; break;
2647 case CPP_AND: if (b == a) return CPP_AND_AND; break;
2648 case CPP_OR: if (b == a) return CPP_OR_OR; break;
2651 if (b == a) return CPP_MINUS_MINUS;
2652 if (b == CPP_GREATER) return CPP_DEREF;
2655 if (b == a && cxx) return CPP_SCOPE;
2656 if (b == CPP_GREATER && CPP_OPTION (pfile, digraphs))
2657 {*digraph = 1; return CPP_CLOSE_SQUARE;} /* :> digraph */
2661 if (CPP_OPTION (pfile, digraphs))
2663 if (b == CPP_GREATER)
2664 {*digraph = 1; return CPP_CLOSE_BRACE;} /* %> digraph */
2666 {*digraph = 1; return CPP_HASH;} /* %: digraph */
2670 if (b == CPP_MULT && cxx) return CPP_DEREF_STAR;
2673 if (b == CPP_MULT && cxx) return CPP_DOT_STAR;
2674 if (b == CPP_NUMBER) return CPP_NUMBER;
2678 if (b == a && (token1->flags & DIGRAPH) == (token2->flags & DIGRAPH))
2680 {*digraph = (token1->flags & DIGRAPH); return CPP_PASTE;}
2684 if (b == CPP_NAME) return CPP_NAME;
2686 && is_numstart(token2->val.str.text[0])) return CPP_NAME;
2688 && token1->val.node == pfile->spec_nodes->n_L) return CPP_WCHAR;
2690 && token1->val.node == pfile->spec_nodes->n_L) return CPP_WSTRING;
2694 if (b == CPP_NUMBER) return CPP_NUMBER;
2695 if (b == CPP_NAME) return CPP_NUMBER;
2696 if (b == CPP_DOT) return CPP_NUMBER;
2697 /* Numbers cannot have length zero, so this is safe. */
2698 if ((b == CPP_PLUS || b == CPP_MINUS)
2699 && VALID_SIGN ('+', token1->val.str.text[token1->val.str.len - 1]))
2710 /* Check if TOKEN is to be ##-pasted with the token after it. */
2711 static const cpp_token *
2712 maybe_paste_with_next (pfile, token)
2714 const cpp_token *token;
2717 const cpp_token *second;
2718 cpp_context *context = CURRENT_CONTEXT (pfile);
2720 /* Is this token on the LHS of ## ? */
2722 while ((token->flags & PASTE_LEFT)
2723 || ((context->flags & CONTEXT_PASTEL)
2724 && context->posn == context->count))
2726 /* Suppress macro expansion for next token, but don't conflict
2727 with the other method of suppression. If it is an argument,
2728 macro expansion within the argument will still occur. */
2729 pfile->paste_level = pfile->cur_context;
2730 second = _cpp_get_token (pfile);
2731 pfile->paste_level = 0;
2733 /* Ignore placemarker argument tokens (cannot be from an empty
2734 macro since macros are not expanded). */
2735 if (token->type == CPP_PLACEMARKER)
2736 pasted = duplicate_token (pfile, second);
2737 else if (second->type == CPP_PLACEMARKER)
2739 /* GCC has special extended semantics for , ## b where b is
2740 a varargs parameter: the comma disappears if b was given
2741 no actual arguments (not merely if b is an empty
2743 if (token->type == CPP_COMMA && second->flags & VOID_REST)
2744 pasted = duplicate_token (pfile, second);
2746 pasted = duplicate_token (pfile, token);
2751 enum cpp_ttype type = can_paste (pfile, token, second, &digraph);
2753 if (type == CPP_EOF)
2755 if (CPP_OPTION (pfile, warn_paste))
2757 /* Do not complain about , ## <whatever> if
2758 <whatever> came from a variable argument, because
2759 the author probably intended the ## to trigger
2760 the special extended semantics (see above). */
2761 if (token->type == CPP_COMMA
2762 && IS_ARG_CONTEXT (CURRENT_CONTEXT (pfile))
2763 && ON_REST_ARG (CURRENT_CONTEXT (pfile) - 1))
2767 "pasting would not give a valid preprocessing token");
2769 _cpp_push_token (pfile, second);
2773 if (type == CPP_NAME || type == CPP_NUMBER)
2775 /* Join spellings. */
2778 pasted = get_temp_token (pfile);
2779 buf = (U_CHAR *) alloca (TOKEN_LEN (token) + TOKEN_LEN (second));
2780 end = spell_token (pfile, token, buf);
2781 end = spell_token (pfile, second, end);
2784 if (type == CPP_NAME)
2785 pasted->val.node = cpp_lookup (pfile, buf, end - buf);
2788 pasted->val.str.text = uxstrdup (buf);
2789 pasted->val.str.len = end - buf;
2792 else if (type == CPP_WCHAR || type == CPP_WSTRING)
2793 pasted = duplicate_token (pfile, second);
2796 pasted = get_temp_token (pfile);
2797 pasted->val.integer = 0;
2800 pasted->type = type;
2801 pasted->flags = digraph ? DIGRAPH : 0;
2803 if (type == CPP_NAME && pasted->val.node->type == T_OPERATOR)
2805 pasted->type = pasted->val.node->value.code;
2806 pasted->flags |= NAMED_OP;
2810 /* The pasted token gets the whitespace flags and position of the
2811 first token, the PASTE_LEFT flag of the second token, plus the
2812 PASTED flag to indicate it is the result of a paste. However, we
2813 want to preserve the DIGRAPH flag. */
2814 pasted->flags &= ~(PREV_WHITE | BOL | PASTE_LEFT);
2815 pasted->flags |= ((token->flags & (PREV_WHITE | BOL))
2816 | (second->flags & PASTE_LEFT) | PASTED);
2817 pasted->col = token->col;
2818 pasted->line = token->line;
2820 /* See if there is another token to be pasted onto the one we just
2823 context = CURRENT_CONTEXT (pfile);
2829 /* Convert a token sequence to a single string token according to the
2830 rules of the ISO C #-operator. */
2831 #define INIT_SIZE 200
2833 stringify_arg (pfile, token)
2835 const cpp_token *token;
2838 unsigned char *main_buf;
2839 unsigned int prev_value, backslash_count = 0;
2840 unsigned int buf_used = 0, whitespace = 0, buf_cap = INIT_SIZE;
2842 push_arg_context (pfile, token);
2843 prev_value = prevent_macro_expansion (pfile);
2844 main_buf = (unsigned char *) xmalloc (buf_cap);
2846 result = get_temp_token (pfile);
2847 ASSIGN_FLAGS_AND_POS (result, token);
2849 for (; (token = _cpp_get_token (pfile))->type != CPP_EOF; )
2853 unsigned int len = TOKEN_LEN (token);
2855 if (token->type == CPP_PLACEMARKER)
2858 escape = (token->type == CPP_STRING || token->type == CPP_WSTRING
2859 || token->type == CPP_CHAR || token->type == CPP_WCHAR);
2863 if (buf_used + len > buf_cap)
2865 buf_cap = buf_used + len + INIT_SIZE;
2866 main_buf = xrealloc (main_buf, buf_cap);
2869 if (whitespace && (token->flags & PREV_WHITE))
2870 main_buf[buf_used++] = ' ';
2873 buf = (unsigned char *) xmalloc (len);
2875 buf = main_buf + buf_used;
2877 len = spell_token (pfile, token, buf) - buf;
2880 buf_used = quote_string (&main_buf[buf_used], buf, len) - main_buf;
2887 if (token->type == CPP_BACKSLASH)
2890 backslash_count = 0;
2893 /* Ignore the final \ of invalid string literals. */
2894 if (backslash_count & 1)
2896 cpp_warning (pfile, "invalid string literal, ignoring final '\\'");
2900 result->type = CPP_STRING;
2901 result->val.str.text = main_buf;
2902 result->val.str.len = buf_used;
2903 restore_macro_expansion (pfile, prev_value);
2907 /* Allocate more room on the context stack of PFILE. */
2909 expand_context_stack (pfile)
2912 pfile->context_cap += pfile->context_cap + 20;
2913 pfile->contexts = (cpp_context *)
2914 xrealloc (pfile->contexts, pfile->context_cap * sizeof (cpp_context));
2917 /* Push the context of macro NODE onto the context stack. TOKEN is
2918 the CPP_NAME token invoking the macro. */
2920 push_macro_context (pfile, token)
2922 const cpp_token *token;
2924 unsigned char orig_flags;
2926 cpp_context *context;
2927 cpp_hashnode *node = token->val.node;
2929 /* Token's flags may change when parsing args containing a nested
2930 invocation of this macro. */
2931 orig_flags = token->flags & (PREV_WHITE | BOL);
2933 if (node->value.expansion->paramc >= 0)
2935 unsigned int error, prev_nme;
2937 /* Allocate room for the argument contexts, and parse them. */
2938 args = (macro_args *) xmalloc (sizeof (macro_args));
2939 args->ends = (unsigned int *)
2940 xmalloc (node->value.expansion->paramc * sizeof (unsigned int));
2944 args->level = pfile->cur_context;
2946 prev_nme = prevent_macro_expansion (pfile);
2948 error = parse_args (pfile, node, args);
2950 restore_macro_expansion (pfile, prev_nme);
2953 free_macro_args (args);
2958 /* Now push its context. */
2959 pfile->cur_context++;
2960 if (pfile->cur_context == pfile->context_cap)
2961 expand_context_stack (pfile);
2963 context = CURRENT_CONTEXT (pfile);
2964 context->u.list = node->value.expansion;
2965 context->args = args;
2967 context->count = context->u.list->tokens_used;
2968 context->level = pfile->cur_context;
2970 context->pushed_token = 0;
2972 /* Set the flags of the first token. We know there must
2973 be one, empty macros are a single placemarker token. */
2974 MODIFY_FLAGS_AND_POS (&context->u.list->tokens[0], token, orig_flags);
2979 /* Push an argument to the current macro onto the context stack.
2980 TOKEN is the MACRO_ARG token representing the argument expansion. */
2982 push_arg_context (pfile, token)
2984 const cpp_token *token;
2986 cpp_context *context;
2989 pfile->cur_context++;
2990 if (pfile->cur_context == pfile->context_cap)
2991 expand_context_stack (pfile);
2993 context = CURRENT_CONTEXT (pfile);
2994 args = context[-1].args;
2996 context->count = token->val.aux ? args->ends[token->val.aux - 1]: 0;
2997 context->u.arg = args->tokens + context->count;
2998 context->count = args->ends[token->val.aux] - context->count;
3001 context->level = args->level;
3002 context->flags = CONTEXT_ARG | CONTEXT_RAW;
3003 context->pushed_token = 0;
3005 /* Set the flags of the first token. There is one. */
3007 const cpp_token *first = context->u.arg[0];
3009 first = context->u.arg[1];
3011 MODIFY_FLAGS_AND_POS ((cpp_token *) first, token,
3012 token->flags & (PREV_WHITE | BOL));
3015 if (token->flags & PASTE_LEFT)
3016 context->flags |= CONTEXT_PASTEL;
3017 if (pfile->paste_level)
3018 context->flags |= CONTEXT_PASTER;
3021 /* "Unget" a token. It is effectively inserted in the token queue and
3022 will be returned by the next call to get_raw_token. */
3024 _cpp_push_token (pfile, token)
3026 const cpp_token *token;
3028 cpp_context *context = CURRENT_CONTEXT (pfile);
3030 if (context->posn > 0)
3032 const cpp_token *prev;
3033 if (IS_ARG_CONTEXT (context))
3034 prev = context->u.arg[context->posn - 1];
3036 prev = &context->u.list->tokens[context->posn - 1];
3045 if (context->pushed_token)
3046 cpp_ice (pfile, "two tokens pushed in a row");
3047 if (token->type != CPP_EOF)
3048 context->pushed_token = token;
3049 /* Don't push back a directive's CPP_EOF, step back instead. */
3050 else if (pfile->cur_context == 0)
3051 pfile->contexts[0].posn--;
3054 /* Handle a preprocessing directive. TOKEN is the CPP_HASH token
3055 introducing the directive. */
3057 process_directive (pfile, token)
3059 const cpp_token *token;
3061 const struct directive *d = pfile->token_list.directive;
3064 /* Skip over the directive name. */
3065 if (token[1].type == CPP_NAME)
3066 _cpp_get_raw_token (pfile);
3067 else if (token[1].type != CPP_NUMBER)
3068 cpp_ice (pfile, "directive begins with %s?!", TOKEN_NAME (token));
3070 if (! (d->flags & EXPAND))
3071 prev_nme = prevent_macro_expansion (pfile);
3072 (void) (*d->handler) (pfile);
3073 if (! (d->flags & EXPAND))
3074 restore_macro_expansion (pfile, prev_nme);
3075 _cpp_skip_rest_of_line (pfile);
3078 /* The external interface to return the next token. All macro
3079 expansion and directive processing is handled internally, the
3080 caller only ever sees the output after preprocessing. */
3082 cpp_get_token (pfile)
3085 const cpp_token *token;
3086 /* Loop till we hit a non-directive, non-placemarker token. */
3089 token = _cpp_get_token (pfile);
3091 if (token->type == CPP_PLACEMARKER)
3094 if (token->type == CPP_HASH && token->flags & BOL
3095 && pfile->token_list.directive)
3097 process_directive (pfile, token);
3105 /* The internal interface to return the next token. There are two
3106 differences between the internal and external interfaces: the
3107 internal interface may return a PLACEMARKER token, and it does not
3108 process directives. */
3110 _cpp_get_token (pfile)
3113 const cpp_token *token, *old_token;
3116 /* Loop until we hit a non-macro token. */
3119 token = get_raw_token (pfile);
3121 /* Short circuit EOF. */
3122 if (token->type == CPP_EOF)
3125 /* If we are skipping... */
3126 if (pfile->skipping)
3128 /* we still have to process directives, */
3129 if (pfile->token_list.directive)
3132 /* but everything else is ignored. */
3133 _cpp_skip_rest_of_line (pfile);
3137 /* If there's a potential control macro and we get here, then that
3138 #ifndef didn't cover the entire file and its argument shouldn't
3139 be taken as a control macro. */
3140 pfile->potential_control_macro = 0;
3144 /* See if there's a token to paste with this one. */
3145 if (!pfile->paste_level)
3146 token = maybe_paste_with_next (pfile, token);
3148 /* If it isn't a macro, return it now. */
3149 if (token->type != CPP_NAME || token->val.node->type == T_VOID)
3152 /* Is macro expansion disabled in general, or are we in the
3153 middle of a token paste, or was this token just pasted?
3154 (Note we don't check token->flags & PASTED, because that
3155 counts tokens that were pasted at some point in the past,
3156 we're only interested in tokens that were pasted by this call
3157 to maybe_paste_with_next.) */
3158 if (pfile->no_expand_level == pfile->cur_context
3159 || pfile->paste_level
3160 || (token != old_token
3161 && pfile->no_expand_level + 1 == pfile->cur_context))
3164 node = token->val.node;
3165 if (node->type != T_MACRO)
3166 return special_symbol (pfile, node, token);
3168 if (is_macro_disabled (pfile, node->value.expansion, token))
3171 if (pfile->cur_context > CPP_STACK_MAX)
3173 cpp_error (pfile, "macros nested too deep invoking '%s'", node->name);
3177 if (push_macro_context (pfile, token))
3183 /* Returns the next raw token, i.e. without performing macro
3184 expansion. Argument contexts are automatically entered. */
3185 static const cpp_token *
3186 get_raw_token (pfile)
3189 const cpp_token *result;
3190 cpp_context *context;
3194 context = CURRENT_CONTEXT (pfile);
3195 if (context->pushed_token)
3197 result = context->pushed_token;
3198 context->pushed_token = 0;
3199 return result; /* Cannot be a CPP_MACRO_ARG */
3201 else if (context->posn == context->count)
3203 if (pop_context (pfile))
3207 else if (IS_ARG_CONTEXT (context))
3209 result = context->u.arg[context->posn++];
3212 context->flags ^= CONTEXT_RAW;
3213 result = context->u.arg[context->posn++];
3215 return result; /* Cannot be a CPP_MACRO_ARG */
3218 result = &context->u.list->tokens[context->posn++];
3220 if (result->type != CPP_MACRO_ARG)
3223 if (result->flags & STRINGIFY_ARG)
3224 return stringify_arg (pfile, result);
3226 push_arg_context (pfile, result);
3230 /* Internal interface to get the token without macro expanding. */
3232 _cpp_get_raw_token (pfile)
3235 int prev_nme = prevent_macro_expansion (pfile);
3236 const cpp_token *result = _cpp_get_token (pfile);
3237 restore_macro_expansion (pfile, prev_nme);
3241 /* A thin wrapper to lex_line. CLEAR is non-zero if the current token
3242 list should be overwritten, or zero if we need to append
3243 (typically, if we are within the arguments to a macro, or looking
3244 for the '(' to start a function-like macro invocation). */
3246 lex_next (pfile, clear)
3250 cpp_toklist *list = &pfile->token_list;
3251 const cpp_token *old_list = list->tokens;
3252 unsigned int old_used = list->tokens_used;
3256 /* Release all temporary tokens. */
3257 _cpp_clear_toklist (list);
3258 pfile->contexts[0].posn = 0;
3259 if (pfile->temp_used)
3260 release_temp_tokens (pfile);
3262 lex_line (pfile, list);
3263 pfile->contexts[0].count = list->tokens_used;
3265 if (!clear && pfile->args)
3267 /* Fix up argument token pointers. */
3268 if (old_list != list->tokens)
3272 for (i = 0; i < pfile->args->used; i++)
3274 const cpp_token *token = pfile->args->tokens[i];
3275 if (token >= old_list && token < old_list + old_used)
3276 pfile->args->tokens[i] = (const cpp_token *)
3277 ((char *) token + ((char *) list->tokens - (char *) old_list));
3281 /* 6.10.3 paragraph 11: If there are sequences of preprocessing
3282 tokens within the list of arguments that would otherwise act as
3283 preprocessing directives, the behavior is undefined.
3285 This implementation will report a hard error and treat the
3286 'sequence of preprocessing tokens' as part of the macro argument,
3289 Note if pfile->args == 0, we're OK since we're only inside a
3290 macro argument after a '('. */
3291 if (list->directive)
3293 cpp_error_with_line (pfile, list->tokens[old_used].line,
3294 list->tokens[old_used].col,
3295 "#%s may not be used inside a macro argument",
3296 list->directive->name);
3304 /* Pops a context off the context stack. If we're at the bottom, lexes
3305 the next logical line. Returns EOF if we're at the end of the
3306 argument list to the # operator, or we should not "overflow"
3307 into the rest of the file (e.g. 6.10.3.1.1). */
3312 cpp_context *context;
3314 if (pfile->cur_context == 0)
3316 /* If we are currently processing a directive, do not advance. 6.10
3317 paragraph 2: A new-line character ends the directive even if it
3318 occurs within what would otherwise be an invocation of a
3319 function-like macro. */
3320 if (pfile->token_list.directive)
3323 return lex_next (pfile, pfile->no_expand_level == UINT_MAX);
3326 /* Argument contexts, when parsing args or handling # operator
3327 return CPP_EOF at the end. */
3328 context = CURRENT_CONTEXT (pfile);
3329 if (IS_ARG_CONTEXT (context) && pfile->cur_context == pfile->no_expand_level)
3332 /* Free resources when leaving macro contexts. */
3334 free_macro_args (context->args);
3336 if (pfile->cur_context == pfile->no_expand_level)
3337 pfile->no_expand_level--;
3338 pfile->cur_context--;
3343 /* Turn off macro expansion at the current context level. */
3345 prevent_macro_expansion (pfile)
3348 unsigned int prev_value = pfile->no_expand_level;
3349 pfile->no_expand_level = pfile->cur_context;
3353 /* Restore macro expansion to its previous state. */
3355 restore_macro_expansion (pfile, prev_value)
3357 unsigned int prev_value;
3359 pfile->no_expand_level = prev_value;
3362 /* Used by cpperror.c to obtain the correct line and column to report
3365 _cpp_get_line (pfile, pcol)
3370 const cpp_token *cur_token;
3372 if (pfile->in_lex_line)
3373 index = pfile->token_list.tokens_used;
3375 index = pfile->contexts[0].posn;
3384 cur_token = &pfile->token_list.tokens[index - 1];
3386 *pcol = cur_token->col;
3387 return cur_token->line;
3390 #define DSC(str) (const U_CHAR *)str, sizeof str - 1
3391 static const char * const monthnames[] =
3393 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3394 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
3397 /* Handle builtin macros like __FILE__. */
3398 static const cpp_token *
3399 special_symbol (pfile, node, token)
3402 const cpp_token *token;
3414 ip = CPP_BUFFER (pfile);
3419 if (node->type == T_BASE_FILE)
3420 while (CPP_PREV_BUFFER (ip) != NULL)
3421 ip = CPP_PREV_BUFFER (ip);
3423 file = ip->nominal_fname;
3425 result = make_string_token (get_temp_token (pfile), (U_CHAR *) file,
3430 case T_INCLUDE_LEVEL:
3431 /* pfile->include_depth counts the primary source as level 1,
3432 but historically __INCLUDE_DEPTH__ has called the primary
3434 result = alloc_number_token (pfile, pfile->include_depth - 1);
3438 /* If __LINE__ is embedded in a macro, it must expand to the
3439 line of the macro's invocation, not its definition.
3440 Otherwise things like assert() will not work properly. */
3441 result = alloc_number_token (pfile, _cpp_get_line (pfile, NULL));
3448 #ifdef STDC_0_IN_SYSTEM_HEADERS
3449 if (CPP_IN_SYSTEM_HEADER (pfile)
3450 && pfile->spec_nodes->n__STRICT_ANSI__->type == T_VOID)
3453 result = alloc_number_token (pfile, stdc);
3459 if (pfile->date == 0)
3461 /* Allocate __DATE__ and __TIME__ from permanent storage,
3462 and save them in pfile so we don't have to do this again.
3463 We don't generate these strings at init time because
3464 time() and localtime() are very slow on some systems. */
3465 time_t tt = time (NULL);
3466 struct tm *tb = localtime (&tt);
3468 pfile->date = make_string_token
3469 ((cpp_token *) xmalloc (sizeof (cpp_token)), DSC("Oct 11 1347"));
3470 pfile->time = make_string_token
3471 ((cpp_token *) xmalloc (sizeof (cpp_token)), DSC("12:34:56"));
3473 sprintf ((char *) pfile->date->val.str.text, "%s %2d %4d",
3474 monthnames[tb->tm_mon], tb->tm_mday, tb->tm_year + 1900);
3475 sprintf ((char *) pfile->time->val.str.text, "%02d:%02d:%02d",
3476 tb->tm_hour, tb->tm_min, tb->tm_sec);
3478 result = node->type == T_DATE ? pfile->date: pfile->time;
3482 cpp_error (pfile, "attempt to use poisoned \"%s\"", node->name);
3486 cpp_ice (pfile, "invalid special hash type");
3490 ASSIGN_FLAGS_AND_POS (result, token);
3495 /* Allocate pfile->input_buffer, and initialize trigraph_map[]
3496 if it hasn't happened already. */
3499 _cpp_init_input_buffer (pfile)
3504 init_trigraph_map ();
3505 _cpp_init_toklist (&pfile->token_list, DUMMY_TOKEN);
3506 pfile->no_expand_level = UINT_MAX;
3507 pfile->context_cap = 20;
3508 pfile->cur_context = 0;
3510 pfile->contexts = (cpp_context *)
3511 xmalloc (pfile->context_cap * sizeof (cpp_context));
3513 /* Clear the base context. */
3514 base = &pfile->contexts[0];
3515 base->u.list = &pfile->token_list;
3521 base->pushed_token = 0;
3524 /* Moves to the end of the directive line, popping contexts as
3527 _cpp_skip_rest_of_line (pfile)
3530 /* Discard all stacked contexts. */
3532 for (i = pfile->cur_context; i > 0; i--)
3533 if (pfile->contexts[i].args)
3534 free_macro_args (pfile->contexts[i].args);
3536 if (pfile->no_expand_level <= pfile->cur_context)
3537 pfile->no_expand_level = 0;
3538 pfile->cur_context = 0;
3540 /* Clear the base context, and clear the directive pointer so that
3541 get_raw_token will advance to the next line. */
3542 pfile->contexts[0].count = 0;
3543 pfile->contexts[0].posn = 0;
3544 pfile->token_list.directive = 0;
3547 /* Directive handler wrapper used by the command line option
3550 _cpp_run_directive (pfile, dir, buf, count)
3552 const struct directive *dir;
3556 if (cpp_push_buffer (pfile, (const U_CHAR *)buf, count) != NULL)
3558 unsigned int prev_lvl = 0;
3560 /* Scan the line now, else prevent_macro_expansion won't work. */
3561 lex_next (pfile, 1);
3562 if (! (dir->flags & EXPAND))
3563 prev_lvl = prevent_macro_expansion (pfile);
3565 (void) (*dir->handler) (pfile);
3567 if (! (dir->flags & EXPAND))
3568 restore_macro_expansion (pfile, prev_lvl);
3570 _cpp_skip_rest_of_line (pfile);
3571 cpp_pop_buffer (pfile);