为flex定义的C令牌文件？

Question

我想将C文件分割成标记，而不是编译而是分析。我觉得这应该是非常直接的，并尝试在线寻找定义为tokens.l（或类似的文件）的flex，所有C语法已经定义，但找不到任何东西。我想知道是否有任何形式的定义的语法浮动，或者如果我可能会这样做是错误的？

Answer 1

是的，至少有one左右。

编辑：

由于有一些不处理的几个问题，也许这是值得一些（手写）词法代码，我几年前写的。这基本上只处理翻译的阶段1，2和3。如果你定义了DIGRAPH，它也会打开一些代码来翻译C++二元图。但是，如果内存服务，那么它在翻译的时候就会比实际发生的事情要早，但是在任何情况下你都不需要它。 OTOH，这甚至不会尝试识别任何接近所有标记的地方 - 大多数情况下它将源分成注释，字符文本，字符串文本以及几乎所有的东西。 OTOH，它可以处理三字母，行拼接等。

我想我还应该补充一点，这会让平台的行尾字符转换为底线实现的换行符，方法是打开翻译后的文本）模式。在大多数情况下，这可能是正确的，但如果你想产生一个类似于交叉编译器的东西，你的源文件有不同于此主机正常的行结束顺序，那么你可能需要改变它。

首先定义外部接口，这一切的东西头：如何容易/困难

/* get_src.c */ 
#include <stdio.h> 
#include <string.h> 
#include <ctype.h> 
#include <stdlib.h> 

#define GET_SOURCE 
#include "get_src.h" 

static size_t current = 0; 

char last_token[MAX_TOKEN_SIZE]; 

PFILE *parse_fopen(char const *name) { 

    PFILE *temp = malloc(sizeof(PFILE)); 

    if (NULL != temp) { 
     temp->file = fopen(name, "r"); 
     memset(temp->peeks, 0, sizeof(temp->peeks)); 
     temp->last_peek = 0; 
    } 
    return temp; 
} 

PFILE *parse_ffopen(FILE *file) { 

    PFILE *temp = malloc(sizeof(PFILE)); 

    if (NULL != temp) { 
     temp->file = file; 
     memset(temp->peeks, 0, sizeof(temp->peeks)); 
     temp->last_peek = 0; 
    } 
    return temp; 
} 

int parse_fclose(PFILE *stream) { 

    int retval = fclose(stream->file); 

    free(stream); 
    return retval; 
} 

static void addchar(int ch) { 
/* adds the passed character to the end of `last_token' */ 

    if (current < sizeof(last_token) -1) 
     last_token[current++] = (char)ch; 

    if (current == sizeof(last_token)-1) 
     last_token[current] = '\0'; 
} 

static void clear(void) { 
/* clears the previous token and starts building a new one. */ 
    current = 0; 
} 

static int read_char(PFILE *stream) { 
    if (stream->last_peek > 0) 
     return stream->peeks[--stream->last_peek]; 
    return fgetc(stream->file); 
} 

void unget_character(int ch, PFILE * stream) { 
    if (stream->last_peek < sizeof(stream->peeks)) 
     stream->peeks[stream->last_peek++] = ch; 
} 

static int check_trigraph(PFILE *stream) { 
/* Checks for trigraphs and returns the equivalant character if there 
* is one. Expects that the leading '?' of the trigraph has already 
* been read before this is called. 
*/ 

    int ch; 

    if ('?' != (ch=read_char(stream))) { 
     unget_character(ch, stream); 
     return '?'; 
    } 

    ch = read_char(stream); 

    switch(ch) { 
     case '(': return '['; 
     case ')': return ']'; 
     case '/': return '\\'; 
     case '\'': return '^'; 
     case '<': return '{'; 
     case '>': return '}'; 
     case '!': return '|'; 
     case '-': return '~'; 
     case '=': return '#'; 
     default: 
      unget_character('?', stream); 
      unget_character(ch, stream); 
      return '?'; 
    } 
} 

#ifdef DIGRAPH 
static int check_digraph(PFILE *stream, int first) { 
/* Checks for a digraph. The first character of the digraph is 
* transmitted as the second parameter, as there are several possible 
* first characters of a digraph. 
*/ 

    int ch = read_char(stream); 

    switch(first) { 
     case '<': 
      if ('%' == ch) 
       return '{'; 
      if (':' == ch) 
       return '['; 
      break; 
     case ':': 
      if ('>' == ch) 
       return ']'; 
      break; 
     case '%': 
      if ('>' == ch) 
       return '}'; 
      if (':' == ch) 
       return '#'; 
      break; 
    } 

/* If it's not one of the specific combos above, return the characters 
* separately and unchanged by putting the second one back into the 
* stream, and returning the first one as-is. 
*/ 
    unget_character(ch, stream); 
    return first; 
} 
#endif 


static int get_char(PFILE *stream) { 
/* Gets a single character from the stream with any trigraphs or digraphs converted 
* to the single character represented. Note that handling digraphs this early in 
* translation isn't really correct (and shouldn't happen in C at all). 
*/ 
    int ch = read_char(stream); 

    if (ch == '?') 
     return check_trigraph(stream); 

#ifdef DIGRAPH 
    if ((ch == '<' || ch == ':' || ch == '%')) 
     return check_digraph(stream, ch); 
#endif 

    return ch; 
} 

int get_character(PFILE *stream) { 
/* gets a character from `stream'. Any amount of any kind of whitespace 
* is returned as a single space. Escaped new-lines are "eaten" here as well. 
*/ 
    int ch; 

    if (!isspace(ch=get_char(stream)) && ch != '\\') 
     return ch; 

    // handle line-slicing 
    if (ch == '\\') { 
     ch = get_char(stream); 
     if (ch == '\n') 
      ch = get_char(stream); 
     else { 
      unget_character(ch, stream); 
      return ch; 
     } 
    } 

    /* If it's a space, skip over consecutive white-space */ 
    while (isspace(ch) && ('\n' != ch)) 
     ch = get_char(stream); 

    if ('\n' == ch) 
     return ch; 

    /* Then put the non-ws character back */ 
    unget_character(ch, stream); 

    /* and return a single space character... */ 
    return ' '; 
} 

static int read_char_lit(PFILE *stream) { 
/* This is used internally by `get_source' (below) - it expects the 
* opening quote of a character literal to have already been read and 
* returns CHAR_LIT or ERROR if there's a newline before a close 
* quote is found, or if the character literal contains more than two 
* characters after escapes are taken into account. 
*/ 

    int ch; 
    int i; 


    clear(); 
    addchar('\''); 

    for (i=0; i<2 && ('\'' != (ch = read_char(stream))); i++) { 

     addchar(ch); 

     if (ch == '\n') 
      return ERROR; 

     if (ch == '\\') { 
      ch = get_char(stream); 
      addchar(ch); 
     } 
    } 
    addchar('\''); 
    addchar('\0'); 

    if (i > 2) 
     return ERROR; 

    return CHAR_LIT; 
} 

static int read_str_lit(PFILE *stream) { 
/* Used internally by get_source. Expects the opening quote of a string 
* literal to have already been read. Returns STR_LIT, or ERROR if a 
* un-escaped newline is found before the close quote. 
*/ 

    int ch; 

    clear(); 
    addchar('"'); 

    while ('"' != (ch = get_char(stream))) { 

     if ('\n' == ch || EOF == ch) 
      return ERROR; 

     addchar(ch); 

     if(ch == '\\') { 
      ch = read_char(stream); 
      addchar(ch); 
     } 

    } 

    addchar('"'); 
    addchar('\0'); 

    return STR_LIT; 
} 

static int read_comment(PFILE *stream) { 
/* Skips over a comment in stream. Assumes the leading '/' has already 
* been read and skips over the body. If we're reading C++ source, skips 
* C++ single line comments as well as normal C comments. 
*/ 
    int ch; 

    clear(); 

    ch = get_char(stream); 

    /* Handle a single line comment. 
    */ 
    if ('/' == ch) { 
     addchar('/'); 
     addchar('/'); 

     while ('\n' != (ch = get_char(stream))) 
      addchar(ch);  

     addchar('\0'); 
     return COMMENT; 
    } 

    if ('*' != ch) { 
     unget_character(ch, stream); 
     return '/'; 
    } 

    addchar('/'); 

    do { 
     addchar(ch); 
     while ('*' !=(ch = get_char(stream))) 
      if (EOF == ch) 
       return ERROR; 
      else 
       addchar(ch); 
     addchar(ch); 
    } while ('/' != (ch=get_char(stream))); 

    addchar('/'); 
    addchar('\0'); 

    return COMMENT; 
} 

int get_source(PFILE *stream) { 
/* reads and returns a single "item" from the stream. An "item" is a 
* comment, a literal or a single character after trigraph and possible 
* digraph substitution has taken place. 
*/ 

    int ch = get_character(stream); 

    switch(ch) { 
     case '\'': 
      return read_char_lit(stream); 
     case '"': 
      return read_str_lit(stream); 
     case '/': 
      return read_comment(stream); 
     default: 
      return ch; 
    } 
} 

#ifdef TEST 

int main(int argc, char **argv) { 
    PFILE *f; 
    int ch; 

    if (argc != 2) { 
     fprintf(stderr, "Usage: get_src <filename>\n"); 
     return EXIT_FAILURE; 
    } 

    if (NULL==(f= parse_fopen(argv[1]))) { 
     fprintf(stderr, "Unable to open: %s\n", argv[1]); 
     return EXIT_FAILURE; 
    } 

    while (EOF!=(ch=get_source(f))) 
     if (ch < 0) 
      printf("\n%s\n", last_token); 
     else 
      printf("%c", ch); 
    parse_fclose(f); 
    return 0;  
} 

#endif 

我不知道：

/* get_src.h */ 
#ifndef GET_SRC_INCLUDED 
#define GET_SRC_INCLUDED 

#include <stdio.h> 

#ifdef __cplusplus 
extern "C" { 
#endif 

/* This is the size of the largest token we'll attempt to deal with. If 
* you want to deal with bigger tokens, change this, and recompile 
* get_src.c. Note that an entire comment is treated as a single token, 
* so long comments could overflow this. In case of an overflow, the 
* entire comment will be read as a single token, but the part larger 
* than this will not be stored. 
*/ 
#define MAX_TOKEN_SIZE 8192 

/* `last_token' will contain the text of the most recently read token (comment, 
* string literal, or character literal). 
*/ 
extern char last_token[]; 

/* This is the maximum number of characters that can be put back into a 
* file opened with parse_fopen or parse_fdopen. 
*/ 
#define MAX_UNGETS 5 

#include <limits.h> 
#include <stdio.h> 

typedef struct { 
    FILE *file; 
    char peeks[MAX_UNGETS]; 
    int last_peek; 
} PFILE; 

/* Some codes we return to indicate having found various items in the 
* source code. ERROR is returned to indicate a newline found in the 
* middle of a character or string literal or if a file ends inside a 
* comment, or if a character literal contains more than two characters. 
* 
* Note that this starts at INT_MIN, the most negative number available 
* in an int. This keeps these symbols from conflicting with any 
* characters read from the file. However, one of these could 
* theoretically conflict with EOF. EOF usually -1, and these are far 
* more negative than that. However, officially EOF can be any value 
* less than 0... 
*/ 
enum { 
    ERROR = INT_MIN, 
    COMMENT, 
    CHAR_LIT, 
    STR_LIT 
}; 

/* Opens a file for parsing and returns a pointer to a structure which 
* can be passed to the other functions in the parser/lexer to identify 
* the file being worked with. 
*/ 
PFILE *parse_fopen(char const *name); 

/* This corresponds closely to fdopen - it takes a FILE * as its 
* only parameter, creates a PFILE structure identifying that file, and 
* returns a pointer to that structure. 
*/ 
PFILE *parse_ffopen(FILE *stream); 

/* Corresponds to fclose. 
*/ 
int parse_fclose(PFILE *stream); 

/* returns characters from `stream' read as C source code. String 
* literals, characters literals and comments are each returned as a 
* single code from those above. All strings of any kind of whitespace 
* are returned as a single space character. 
*/ 
int get_source(PFILE *stream); 

/* Basically, these two work just like the normal versions of the same, 
* with the minor exception that unget_character can unget more than one 
* character. 
*/ 
int get_character(PFILE *stream); 
void unget_character(int ch, PFILE *stream); 

#ifdef __cplusplus 
} 
#endif 

#endif 

然后所有的实施虽然我会想起将Flex集成到一个基于Flex的词法分析器中 - 但我似乎记得Flex有一些钩子来定义它用来读取字符的方式，但我从来没有尝试过使用它，所以我可以“对它说得更多了（最终，甚至无法用任何接近确定的话来说它甚至存在）。