Free-form

Free-form style is enabled with configuration re2c:api:style = free-form. In this style interface primitives should be defined as free-form pieces of code with interpolated variables of the form @@{var} or optionally just @@ if there is a single variable. The set of variables is specific to each primitive. Here's how free-form generic API can be defined in terms of pointers cursor, limit, marker and ctxmarker:

/*!re2c


  re2c:YYPEEK       = "*cursor";


  re2c:YYSKIP       = "++cursor;";


  re2c:YYBACKUP     = "marker = cursor;";


  re2c:YYRESTORE    = "cursor = marker;";


  re2c:YYBACKUPCTX  = "ctxmarker = cursor;";


  re2c:YYRESTORECTX = "cursor = ctxmarker;";


  re2c:YYRESTORETAG = "cursor = ${tag};";


  re2c:YYLESSTHAN   = "limit - cursor < @@{len}";


  re2c:YYSTAGP      = "@@{tag} = cursor;";


  re2c:YYSTAGN      = "@@{tag} = NULL;";


  re2c:YYSHIFT      = "cursor += @@{shift};";


  re2c:YYSHIFTSTAG  = "@@{tag} += @@{shift};";
*/

Function-like

Historically function-like style is the default one. It also can be enabled with configuration re2c:api:style = functions. In this style primitives should be defined as functions or macros with parentheses, accepting the necessary arguments. Here's how function-like generic API can be defined in terms of pointers cursor, limit, marker and ctxmarker using preprocessor macros:

#define  YYPEEK()                 *cursor
#define  YYSKIP()                 ++cursor
#define  YYBACKUP()               marker = cursor
#define  YYRESTORE()              cursor = marker
#define  YYBACKUPCTX()            ctxmarker = cursor
#define  YYRESTORECTX()           cursor = ctxmarker
#define  YYRESTORETAG(tag)        cursor = tag
#define  YYLESSTHAN(len)          limit - cursor < len
#define  YYSTAGP(tag)             tag = cursor
#define  YYSTAGN(tag)             tag = NULL
#define  YYSHIFT(shift)           cursor += shift
#define  YYSHIFTSTAG(tag, shift)  tag += shift

code-template ::


      name '=' code-exprs ';'


    | CODE_TEMPLATE ';'


    | '<undefined>' ';'
code-exprs ::


      <EMPTY>


    | code-exprs code-expr
code-expr ::


      STRING


    | VARIABLE


    | optional


    | list
optional ::


      '(' CONDITIONAL '?' code-exprs ')'


    | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')'
list ::


      '[' VARIABLE ':' code-exprs ']'


    | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']'


    | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']'

A code template is a sequence of string literals, variables, optional elements and lists, or a reference to another code template, or a special value <undefined>. Variables are placeholders that are substituted during code generation phase. List variables are special: when expanding list templates, re2c repeats expressions the right hand side of the column a few times, each time replacing occurrences of the list variable with a value specific to this repetition. Lists have optional bounds (negative values are counted from the end, e.g. -1 means the last element). Conditional names start with a dot. Both conditionals and variables may be either local (specific to the given code template) or global (allowed in all code templates). When re2c reads syntax file, it checks that each code template uses only the variables and conditionals that are allowed in it.

For example, the following code template defines if-then-else construct for a C-like language:

code:if_then_else =


    [branch{0}: topindent "if " cond " {" nl


        indent [stmt: stmt] dedent]


    [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl


        indent [stmt: stmt] dedent]


    topindent "}" nl;

Here branch is a list variable: branch{0} expands to the first branch (which is special, as there is no else part), branch{1:-1} expands to all remaining branches (if any). stmt is also a list variable: [stmt: stmt] is a nested list that expands to a list of statements in the body of the current branch. topindent, indent, dedent and nl are global variables, and .cond is a local conditional (their meaning is described below). This code template could produce the following code:

if x {


    // do something
} else if y {


    // do something else
} else {


    // don't do anything
}

Here's a list of all code templates supported by re2c with their local variables and conditionals. Note that a particular definition may, but does not have to use local variables and conditionals. Any unused code templates should be set to <undefined>.

Here's a list of all global variables that are allowed in syntax files:

Here's a list of all global conditionals that are allowed in syntax files:

// re2c $INPUT -o $OUTPUT
#include <assert.h>
// Expect a null-terminated string.
static int lex(const char *YYCURSOR) {


    int count = 0;


    for (;;) {


    /*!re2c


        re2c:define:YYCTYPE = char;


        re2c:yyfill:enable = 0;


        *      { return -1; }


        [\x00] { return count; }


        [a-z]+ { ++count; continue; }


        [ ]+   { continue; }


    */


    }
}
int main() {


    assert(lex("") == 0);


    assert(lex("one two three") == 3);


    assert(lex("f0ur") == -1);


    return 0;
}

// re2c $INPUT -o $OUTPUT
#include <assert.h>
// Expect a null-terminated string.
static int lex(const char *str, unsigned int len) {


    const char *YYCURSOR = str, *YYLIMIT = str + len, *YYMARKER;


    int count = 0;


    for (;;) {


    /*!re2c


        re2c:define:YYCTYPE = char;


        re2c:yyfill:enable = 0;


        re2c:eof = 0;


        str = ['] ([^'\\] | [\\][^])* ['];


        *    { return -1; }


        $    { return count; }


        str  { ++count; continue; }


        [ ]+ { continue; }


    */


    }
}
#define TEST(s, r) assert(lex(s, sizeof(s) - 1) == r)
int main() {


    TEST("", 0);


    TEST("'qu\0tes' 'are' 'fine: \\'' ", 3);


    TEST("'unterminated\\'", -1);


    return 0;
}

// re2c $INPUT -o $OUTPUT
#include <assert.h>
#include <stdlib.h>
#include <string.h>
/*!max:re2c*/
static int lex(const char *str, unsigned int len) {


    // Make a copy of the string with YYMAXFILL zeroes at the end.


    char *buf = (char*) malloc(len + YYMAXFILL);


    memcpy(buf, str, len);


    memset(buf + len, 0, YYMAXFILL);


    const char *YYCURSOR = buf, *YYLIMIT = buf + len + YYMAXFILL;


    int count = 0;
loop:


    /*!re2c


        re2c:api:style = free-form;


        re2c:define:YYCTYPE = char;


        re2c:define:YYFILL = "goto fail;";


        str = ['] ([^'\\] | [\\][^])* ['];


        [\x00] {


            // Check that it is the sentinel, not some unexpected null.


            if (YYCURSOR - 1 == buf + len) goto exit; else goto fail;


        }


        str  { ++count; goto loop; }


        [ ]+ { goto loop; }


        *    { goto fail; }


    */
fail:


    count = -1;
exit:


    free(buf);


    return count;
}
#define TEST(s, r) assert(lex(s, sizeof(s) - 1) == r)
int main() {


    TEST("", 0);


    TEST("'qu\0tes' 'are' 'fine: \\'' ", 3);


    TEST("'unterminated\\'", -1);


    TEST("'unexpected \0 null\\'", -1);


    return 0;
}

// re2c $INPUT -o $OUTPUT
#include <assert.h>
#include <stdlib.h>
#include <string.h>
static int lex(const char *str, unsigned int len) {


    // For the sake of example create a string without terminating null.


    char *buf = (char*) malloc(len);


    memcpy(buf, str, len);


    const char *cur = buf, *lim = buf + len;


    int count = 0;


    for (;;) {


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:api = custom;


        re2c:api:style = free-form;


        re2c:define:YYCTYPE = char;


        re2c:define:YYPEEK = "cur < lim ? *cur : 0";  // fake null


        re2c:define:YYSKIP = "++cur;";


        *      { count = -1; break; }


        [\x00] { break;; }


        [a-z]+ { ++count; continue;; }


        [ ]+   { continue; }


    */


    }


    free(buf);


    return count;
}
#define TEST(s, r) assert(lex(s, sizeof(s) - 1) == r)
int main() {


    TEST("", 0);


    TEST("one two three ", 3);


    TEST("f0ur", -1);


    return 0;
}

               <-- shift -->


             >-A------------B---------C-------------D#-----------E->


             buffer       token    marker         limit,


                                                  cursor
>-A------------B---------C-------------D------------E#->


             buffer,  marker        cursor        limit


             token


               <-- shift -->


             >-A------------B---------C-------------D#--E (EOF)


             buffer       token    marker         limit,


                                                  cursor
>-A------------B---------C-------------D---E#........


             buffer,  marker       cursor limit


             token

// re2c $INPUT -o $OUTPUT
#include <assert.h>
#include <stdio.h>
#include <string.h>
#define BUFSIZE 4095
typedef struct {


    FILE *file;


    char buf[BUFSIZE + 1], *lim, *cur, *mar, *tok; // +1 for sentinel


    int eof;
} Input;
static int fill(Input *in) {


    if (in->eof) return 1;


    const size_t shift = in->tok - in->buf;


    const size_t used = in->lim - in->tok;


    // Error: lexeme too long. In real life could reallocate a larger buffer.


    if (shift < 1) return 2;


    // Shift buffer contents (discard everything up to the current token).


    memmove(in->buf, in->tok, used);


    in->lim -= shift;


    in->cur -= shift;


    in->mar -= shift;


    in->tok -= shift;


    // Fill free space at the end of buffer with new data from file.


    in->lim += fread(in->lim, 1, BUFSIZE - used, in->file);


    in->lim[0] = 0;


    in->eof = in->lim < in->buf + BUFSIZE;


    return 0;
}
static int lex(Input *in) {


    int count = 0;
loop:


        in->tok = in->cur;


    /*!re2c


        re2c:api:style = free-form;


        re2c:define:YYCTYPE = char;


        re2c:define:YYCURSOR = in->cur;


        re2c:define:YYMARKER = in->mar;


        re2c:define:YYLIMIT = in->lim;


        re2c:define:YYFILL = "fill(in) == 0";


        re2c:eof = 0;


        str = ['] ([^'\\] | [\\][^])* ['];


        *    { return -1; }


        $    { return count; }


        str  { ++count; goto loop; }


        [ ]+ { goto loop; }


    */
}
int main() {


    const char *fname = "input";


    const char content[] = "'qu\0tes' 'are' 'fine: \\'' ";


    // Prepare input file: a few times the size of the buffer, containing


    // strings with zeroes and escaped quotes.


    FILE *f = fopen(fname, "w");


    for (int i = 0; i < BUFSIZE; ++i) {


        fwrite(content, 1, sizeof(content) - 1, f);


    }


    fclose(f);


    int count = 3 * BUFSIZE; // number of quoted strings written to file


    // Initialize lexer state: all pointers are at the end of buffer.


    Input in;


    in.file = fopen(fname, "r");


    in.cur = in.mar = in.tok = in.lim = in.buf + BUFSIZE;


    in.eof = 0;


    // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL.


    in.lim[0] = 0;


    // Run the lexer.


    assert(lex(&in) == count);


    // Cleanup: remove input file.


    fclose(in.file);


    remove(fname);


    return 0;
}

               <-- shift -->                 <-- need -->


             >-A------------B---------C-----D-------E---F--------G->


             buffer       token    marker cursor  limit
>-A------------B---------C-----D-------E---F--------G->


             buffer,  marker cursor               limit


             token


               <-- shift -->                 <-- need -->


             >-A------------B---------C-----D-------E-F        (EOF)


             buffer       token    marker cursor  limit
>-A------------B---------C-----D-------E-F###############


             buffer,  marker cursor                   limit


             token                        <- YYMAXFILL ->

// re2c $INPUT -o $OUTPUT
#include <assert.h>
#include <stdio.h>
#include <string.h>
/*!max:re2c*/
#define BUFSIZE (4096 - YYMAXFILL)
typedef struct {


    FILE *file;


    char buf[BUFSIZE + YYMAXFILL], *lim, *cur, *tok;


    int eof;
} Input;
static int fill(Input *in, size_t need) {


    if (in->eof) return 1;


    const size_t shift = in->tok - in->buf;


    const size_t used = in->lim - in->tok;


    // Error: lexeme too long. In real life could reallocate a larger buffer.


    if (shift < need) return 2;


    // Shift buffer contents (discard everything up to the current token).


    memmove(in->buf, in->tok, used);


    in->lim -= shift;


    in->cur -= shift;


    in->tok -= shift;


    // Fill free space at the end of buffer with new data from file.


    in->lim += fread(in->lim, 1, BUFSIZE - used, in->file);


    // If read less than expected, this is end of input => add zero padding


    // so that the lexer can access characters at the end of buffer.


    if (in->lim < in->buf + BUFSIZE) {


        in->eof = 1;


        memset(in->lim, 0, YYMAXFILL);


        in->lim += YYMAXFILL;


    }


    return 0;
}
static int lex(Input *in) {


    int count = 0;
loop:


        in->tok = in->cur;


    /*!re2c


        re2c:api:style = free-form;


        re2c:define:YYCTYPE = char;


        re2c:define:YYCURSOR = in->cur;


        re2c:define:YYLIMIT = in->lim;


        re2c:define:YYFILL = "if (fill(in, @@) != 0) return -1;";


        str = ['] ([^'\\] | [\\][^])* ['];


        [\x00] {


            // Check that it is the sentinel, not some unexpected null.


            return in->tok == in->lim - YYMAXFILL ? count : -1;


        }


        str  { ++count; goto loop; }


        [ ]+ { goto loop; }


        *    { return -1; }


    */
}
int main() {


    const char *fname = "input";


    const char content[] = "'qu\0tes' 'are' 'fine: \\'' ";


    // Prepare input file: a few times the size of the buffer, containing


    // strings with zeroes and escaped quotes.


    FILE *f = fopen(fname, "w");


    for (int i = 0; i < BUFSIZE; ++i) {


        fwrite(content, 1, sizeof(content) - 1, f);


    }


    fclose(f);


    int count = 3 * BUFSIZE; // number of quoted strings written to file


    // Initialize lexer state: all pointers are at the end of buffer.


    // This immediately triggers YYFILL, as the check `in->cur < in->lim` fails.


    Input in;


    in.file = fopen(fname, "r");


    in.cur = in.tok = in.lim = in.buf + BUFSIZE;


    in.eof = 0;


    // Run the lexer.


    assert(lex(&in) == count);


    // Cleanup: remove input file.


    fclose(in.file);


    remove(fname);


    return 0;
}

// re2c $INPUT -o $OUTPUT -i
#include <stdint.h>
#include <limits.h>
#include <assert.h>
static const uint64_t ERROR = UINT64_MAX;
#define CHECK(n) if (n > UINT32_MAX) return ERROR;
static uint64_t parse_u32(const char *s) {


    const char *YYCURSOR = s, *YYMARKER;


    uint64_t u = 0;


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:define:YYCTYPE = char;


        end = "\x00";


        '0b' / [01]        { goto bin; }


        "0"                { goto oct; }


        "" / [1-9]         { goto dec; }


        '0x' / [0-9a-fA-F] { goto hex; }


        *                  { return ERROR; }


    */
bin:


    /*!re2c


        end   { return u; }


        [01]  { u = u * 2 + (YYCURSOR[-1] - '0'); CHECK(u); goto bin; }


        *     { return ERROR; }


    */
oct:


    /*!re2c


        end   { return u; }


        [0-7] { u = u * 8 + (YYCURSOR[-1] - '0'); CHECK(u); goto oct; }


        *     { return ERROR; }


    */
dec:


    /*!re2c


        end   { return u; }


        [0-9] { u = u * 10 + (YYCURSOR[-1] - '0'); CHECK(u); goto dec; }


        *     { return ERROR; }


    */
hex:


    /*!re2c


        end   { return u; }


        [0-9] { u = u * 16 + (YYCURSOR[-1] - '0');      CHECK(u); goto hex; }


        [a-f] { u = u * 16 + (YYCURSOR[-1] - 'a' + 10); CHECK(u); goto hex; }


        [A-F] { u = u * 16 + (YYCURSOR[-1] - 'A' + 10); CHECK(u); goto hex; }


        *     { return ERROR; }


    */
}
int main() {


    assert(parse_u32("") == ERROR);


    assert(parse_u32("1234567890") == 1234567890);


    assert(parse_u32("0b1101") == 13);


    assert(parse_u32("0x7Fe") == 2046);


    assert(parse_u32("0644") == 420);


    assert(parse_u32("9999999999") == ERROR);


    return 0;
}

// re2c $INPUT -o $OUTPUT -ci
#include <stdint.h>
#include <limits.h>
#include <assert.h>
static const uint64_t ERROR = UINT64_MAX;
/*!conditions:re2c*/
static uint64_t parse_u32(const char *s) {


    const char *YYCURSOR = s, *YYMARKER;


    int c = yycinit;


    uint64_t u = 0;


    /*!re2c


        re2c:api:style = free-form;


        re2c:define:YYCTYPE = char;


        re2c:define:YYGETCONDITION = "c";


        re2c:define:YYSETCONDITION = "c = @@;";


        re2c:yyfill:enable = 0;


        <*> * { return ERROR; }


        <init> '0b' / [01]        :=> bin


        <init> "0"                :=> oct


        <init> "" / [1-9]         :=> dec


        <init> '0x' / [0-9a-fA-F] :=> hex


        <bin, oct, dec, hex> "\x00" { return u; }


        <bin> [01]  { u = u * 2  + (YYCURSOR[-1] - '0');      goto yyc_bin; }


        <oct> [0-7] { u = u * 8  + (YYCURSOR[-1] - '0');      goto yyc_oct; }


        <dec> [0-9] { u = u * 10 + (YYCURSOR[-1] - '0');      goto yyc_dec; }


        <hex> [0-9] { u = u * 16 + (YYCURSOR[-1] - '0');      goto yyc_hex; }


        <hex> [a-f] { u = u * 16 + (YYCURSOR[-1] - 'a' + 10); goto yyc_hex; }


        <hex> [A-F] { u = u * 16 + (YYCURSOR[-1] - 'A' + 10); goto yyc_hex; }


        <!*> { if (u > UINT32_MAX) return ERROR; }


    */
}
int main() {


    assert(parse_u32("") == ERROR);


    assert(parse_u32("1234567890") == 1234567890);


    assert(parse_u32("0b1101") == 13);


    assert(parse_u32("0x7Fe") == 2046);


    assert(parse_u32("0644") == 420);


    assert(parse_u32("9999999999") == ERROR);


    return 0;
}

// re2c $INPUT -o $OUTPUT -f
#include <assert.h>
#include <stdio.h>
#include <string.h>
#define DEBUG 0
#define LOG(...) if (DEBUG) fprintf(stderr, __VA_ARGS__);
// Use a small buffer to cover the case when a lexeme doesn't fit.
// In real world use a larger buffer.
#define BUFSIZE 10
typedef struct {


    FILE *file;


    char buf[BUFSIZE + 1], *lim, *cur, *mar, *tok;


    int state;
} State;
typedef enum {END, READY, WAITING, BAD_PACKET, BIG_PACKET} Status;
static Status fill(State *st) {


    const size_t shift = st->tok - st->buf;


    const size_t used = st->lim - st->tok;


    const size_t free = BUFSIZE - used;


    // Error: no space. In real life can reallocate a larger buffer.


    if (free < 1) return BIG_PACKET;


    // Shift buffer contents (discard already processed data).


    memmove(st->buf, st->tok, used);


    st->lim -= shift;


    st->cur -= shift;


    st->mar -= shift;


    st->tok -= shift;


    // Fill free space at the end of buffer with new data.


    const size_t read = fread(st->lim, 1, free, st->file);


    st->lim += read;


    st->lim[0] = 0; // append sentinel symbol


    return READY;
}
static Status lex(State *st, unsigned int *recv) {


    char yych;


    /*!getstate:re2c*/


    for (;;) {


        st->tok = st->cur;


    /*!re2c


        re2c:api:style = free-form;


        re2c:define:YYCTYPE = "char";


        re2c:define:YYCURSOR = "st->cur";


        re2c:define:YYMARKER = "st->mar";


        re2c:define:YYLIMIT = "st->lim";


        re2c:define:YYGETSTATE = "st->state";


        re2c:define:YYSETSTATE = "st->state = @@;";


        re2c:define:YYFILL = "return WAITING;";


        re2c:eof = 0;


        packet = [a-z]+[;];


        *      { return BAD_PACKET; }


        $      { return END; }


        packet { *recv = *recv + 1; continue; }


    */


    }
}
void test(const char **packets, Status expect) {


    // Create a "socket" (open the same file for reading and writing).


    const char *fname = "pipe";


    FILE *fw = fopen(fname, "w");


    FILE *fr = fopen(fname, "r");


    setvbuf(fw, NULL, _IONBF, 0);


    setvbuf(fr, NULL, _IONBF, 0);


    // Initialize lexer state: `state` value is -1, all pointers are at the end


    // of buffer.


    State st;


    st.file = fr;


    st.cur = st.mar = st.tok = st.lim = st.buf + BUFSIZE;


    // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL.


    st.lim[0] = 0;


    st.state = -1;


    // Main loop. The buffer contains incomplete data which appears packet by


    // packet. When the lexer needs more input it saves its internal state and


    // returns to the caller which should provide more input and resume lexing.


    Status status;


    unsigned int send = 0, recv = 0;


    for (;;) {


        status = lex(&st, &recv);


        if (status == END) {


            LOG("done: got %u packets\n", recv);


            break;


        } else if (status == WAITING) {


            LOG("waiting...\n");


            if (*packets) {


                LOG("sent packet %u\n", send);


                fprintf(fw, "%s", *packets++);


                ++send;


            }


            status = fill(&st);


            LOG("queue: '%s'\n", st.buf);


            if (status == BIG_PACKET) {


                LOG("error: packet too big\n");


                break;


            }


            assert(status == READY);


        } else {


            assert(status == BAD_PACKET);


            LOG("error: ill-formed packet\n");


            break;


        }


    }


    // Check results.


    assert(status == expect);


    if (status == END) assert(recv == send);


    // Cleanup: remove input file.


    fclose(fw);


    fclose(fr);


    remove(fname);
}
int main() {


    const char *packets1[] = {0};


    const char *packets2[] = {"zero;", "one;", "two;", "three;", "four;", 0};


    const char *packets3[] = {"zer0;", 0};


    const char *packets4[] = {"looooooooooong;", 0};


    test(packets1, END);


    test(packets2, END);


    test(packets3, BAD_PACKET);


    test(packets4, BIG_PACKET);


    return 0;
}

// re2c $INPUT -o $OUTPUT
#include <assert.h>
// This example shows how to combine reusable re2c blocks: two blocks
// ('colors' and 'fish') are merged into one. The 'salmon' rule occurs
// in both blocks; the 'fish' block takes priority because it is used
// earlier. Default rule * occurs in all three blocks; the local (not
// inherited) definition takes priority.
typedef enum { COLOR, FISH, DUNNO } What;
/*!rules:re2c:colors


    *                            { assert(false); }


    "red" | "salmon" | "magenta" { return COLOR; }
*/
/*!rules:re2c:fish


    *                            { assert(false); }


    "haddock" | "salmon" | "eel" { return FISH; }
*/
static What lex(const char *s) {


    const char *YYCURSOR = s, *YYMARKER;


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:define:YYCTYPE = char;


        !use:fish;


        !use:colors;


        * { return DUNNO; }  // overrides inherited '*' rules


    */
}
int main() {


    assert(lex("salmon") == FISH);


    assert(lex("what?") == DUNNO);


    return 0;
}

// re2c $INPUT -o $OUTPUT --input-encoding utf8
#include <assert.h>
#include <stdint.h>
// This example supports multiple input encodings: UTF-8 and UTF-32.
// Both lexers are generated from the same rules block, and the use
// blocks add only encoding-specific configurations.
/*!rules:re2c


    re2c:yyfill:enable = 0;


    "∀x ∃y" { return 0; }


    *       { return 1; }
*/
static int lex_utf8(const uint8_t *s) {


    const uint8_t *YYCURSOR = s, *YYMARKER;


    /*!use:re2c


        re2c:define:YYCTYPE = uint8_t;


        re2c:encoding:utf8 = 1;


    */
}
static int lex_utf32(const uint32_t *s) {


    const uint32_t *YYCURSOR = s, *YYMARKER;


    /*!use:re2c


        re2c:define:YYCTYPE = uint32_t;


        re2c:encoding:utf32 = 1;


    */
}
int main() {


    static const uint8_t s8[] = // UTF-8


        { 0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79 };


    static const uint32_t s32[] = // UTF32


        { 0x00002200, 0x00000078, 0x00000020, 0x00002203, 0x00000079 };


    assert(lex_utf8(s8) == 0);


    assert(lex_utf32(s32) == 0);


    return 0;
}

// re2c $INPUT -o $OUTPUT
#include <assert.h>
#include <stddef.h>
typedef struct { int major, minor, patch; } SemVer;
static int s2n(const char *s, const char *e) { // pre-parsed string to number


    int n = 0;


    for (; s < e; ++s) n = n * 10 + (*s - '0');


    return n;
}
static int lex(const char *str, SemVer *ver) {


    const char *YYCURSOR = str, *YYMARKER;


    // User-defined tag variables that are available in semantic action.


    const char *t1, *t2, *t3, *t4, *t5;


    // Autogenerated tag variables used by the lexer to track tag values.


    /*!stags:re2c format = 'const char *@@;\n'; */


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:define:YYCTYPE = char;


        re2c:tags = 1;


        num = [0-9]+;


        @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\x00] {


            ver->major = s2n(t1, t2);


            ver->minor = s2n(t3, t4);


            ver->patch = t5 != NULL ? s2n(t5, YYCURSOR - 1) : 0;


            return 0;


        }


        * { return 1; }


    */
}
int main() {


    SemVer v;


    assert(lex("23.34", &v) == 0 && v.major == 23 && v.minor == 34 && v.patch == 0);


    assert(lex("1.2.999", &v) == 0 && v.major == 1 && v.minor == 2 && v.patch == 999);


    assert(lex("1.a", &v) == 1);


    return 0;
}

// re2c $INPUT -o $OUTPUT --tags
#include <assert.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>
#include <vector>
#define BUFSIZE 4095
struct Input {


    FILE *file;


    char buf[BUFSIZE + 1], *lim, *cur, *mar, *tok;


    // Tag variables must be part of the lexer state passed to YYFILL.


    // They don't correspond to tags and should be autogenerated by re2c.


    /*!stags:re2c format = 'const char *@@;'; */


    bool eof;
};
struct SemVer { int major, minor, patch; };
static bool operator==(const SemVer &x, const SemVer &y) {


    return x.major == y.major && x.minor == y.minor && x.patch == y.patch;
}
static int s2n(const char *s, const char *e) { // pre-parsed string to number


    int n = 0;


    for (; s < e; ++s) n = n * 10 + (*s - '0');


    return n;
}
static int fill(Input &in) {


    if (in.eof) return 1;


    const size_t shift = in.tok - in.buf;


    const size_t used = in.lim - in.tok;


    // Error: lexeme too long. In real life could reallocate a larger buffer.


    if (shift < 1) return 2;


    // Shift buffer contents (discard everything up to the current token).


    memmove(in.buf, in.tok, used);


    in.lim -= shift;


    in.cur -= shift;


    in.mar -= shift;


    in.tok -= shift;


    // Tag variables need to be shifted like other input positions. The check


    // for non-NULL is only needed if some tags are nested inside of alternative


    // or repetition, so that they can have NULL value.


    /*!stags:re2c format = "if (in.@@) in.@@ -= shift;\n"; */


    // Fill free space at the end of buffer with new data from file.


    in.lim += fread(in.lim, 1, BUFSIZE - used, in.file);


    in.lim[0] = 0;


    in.eof = in.lim < in.buf + BUFSIZE;


    return 0;
}
static bool lex(Input &in, std::vector<SemVer> &vers) {


    // User-defined local variables that store final tag values.


    // They are different from tag variables autogenerated with `stags:re2c`,


    // as they are set at the end of match and used only in semantic actions.


    const char *t1, *t2, *t3, *t4;


    for (;;) {


        in.tok = in.cur;


    /*!re2c


        re2c:eof = 0;


        re2c:api:style = free-form;


        re2c:define:YYCTYPE = char;


        re2c:define:YYCURSOR = in.cur;


        re2c:define:YYMARKER = in.mar;


        re2c:define:YYLIMIT = in.lim;


        re2c:define:YYFILL = "fill(in) == 0";


        re2c:tags:expression = "in.@@";


        num = [0-9]+;


        num @t1 "." @t2 num @t3 ("." @t4 num)? [\n] {


            int major = s2n(in.tok, t1);


            int minor = s2n(t2, t3);


            int patch = t4 != NULL ? s2n(t4, in.cur - 1) : 0;


            SemVer ver = {major, minor, patch};


            vers.push_back(ver);


            continue;


        }


        $ { return true; }


        * { return false; }


    */}
}
int main() {


    const char *fname = "input";


    const SemVer semver = {1, 22, 333};


    std::vector<SemVer> expect(BUFSIZE, semver), actual;


    // Prepare input file (make sure it exceeds buffer size).


    FILE *f = fopen(fname, "w");


    for (int i = 0; i < BUFSIZE; ++i) fprintf(f, "1.22.333\n");


    fclose(f);


    // Reopen input file for reading.


    f = fopen(fname, "r");


    // Initialize lexer state: all pointers are at the end of buffer.


    Input in;


    in.file = f;


    in.cur = in.mar = in.tok = in.lim = in.buf + BUFSIZE;


    /*!stags:re2c format = "in.@@ = in.lim;\n"; */


    in.eof = false;


    // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL.


    *in.lim = 0;


  


    // Run the lexer and check results.


    assert(lex(in, actual) && expect == actual);


    // Cleanup: remove input file.


    fclose(f);


    remove(fname);


    return 0;
}

// re2c $INPUT -o $OUTPUT
#include <assert.h>
#include <stddef.h>
typedef struct { int major, minor, patch; } SemVer;
static int s2n(const char *s, const char *e) { // pre-parsed string to number


    int n = 0;


    for (; s < e; ++s) n = n * 10 + (*s - '0');


    return n;
}
static int lex(const char *str, SemVer *ver) {


    const char *YYCURSOR = str, *YYMARKER;


    // Final tag variables available in semantic action.


    /*!svars:re2c format = 'const char *@@;\n'; */


    // Intermediate tag variables used by the lexer (must be autogenerated).


    /*!stags:re2c format = 'const char *@@;\n'; */


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:define:YYCTYPE = char;


        re2c:captvars = 1;


        num = [0-9]+;


        (num) "." (num) ("." num)? [\x00] {


            (void) yytl0; (void) yytr0; // some variables are unused


            ver->major = s2n(yytl1, yytr1);


            ver->minor = s2n(yytl2, yytr2);


            ver->patch = yytl3 ? s2n(yytl3 + 1, yytr3) : 0;


            return 0;


        }


        * { return 1; }


    */
}
int main() {


    SemVer v;


    assert(lex("23.34", &v) == 0 && v.major == 23 && v.minor == 34 && v.patch == 0);


    assert(lex("1.2.999", &v) == 0 && v.major == 1 && v.minor == 2 && v.patch == 999);


    assert(lex("1.a", &v) == 1);


    return 0;
}

// re2c $INPUT -o $OUTPUT
#include <assert.h>
#include <stddef.h>
#include <vector>
static const int MTAG_ROOT = -1;
// An m-tag tree is a way to store histories with an O(1) copy operation.
// Histories naturally form a tree, as they have common start and fork at some
// point. The tree is stored as an array of pairs (tag value, link to parent).
// An m-tag is represented with a single link in the tree (array index).
struct Mtag {


    const char *elem; // tag value


    int pred; // index of the predecessor node or root
};
typedef std::vector<Mtag> MtagTrie;
typedef std::vector<int> Ver; // unbounded number of version components
static int s2n(const char *s, const char *e) { // pre-parsed string to number


    int n = 0;


    for (; s < e; ++s) n = n * 10 + (*s - '0');


    return n;
}
// Append a single value to an m-tag history.
static void add_mtag(MtagTrie &trie, int &mtag, const char *value) {


    Mtag m = {value, mtag};


    mtag = (int)trie.size();


    trie.push_back(m);
}
// Recursively unwind tag histories and collect version components.
static void unfold(const MtagTrie &trie, int x, int y, Ver &ver) {


    // Reached the root of the m-tag tree, stop recursion.


    if (x == MTAG_ROOT && y == MTAG_ROOT) return;


    // Unwind history further.


    unfold(trie, trie[x].pred, trie[y].pred, ver);


    // Get tag values. Tag histories must have equal length.


    assert(x != MTAG_ROOT && y != MTAG_ROOT);


    const char *ex = trie[x].elem, *ey = trie[y].elem;


    if (ex != NULL && ey != NULL) {


        // Both tags are valid pointers, extract component.


        ver.push_back(s2n(ex, ey));


    } else {


        // Both tags are NULL (this corresponds to zero repetitions).


        assert(ex == NULL && ey == NULL);


    }
}
static bool parse(const char *str, Ver &ver) {


    const char *YYCURSOR = str, *YYMARKER;


    MtagTrie mt;


    // User-defined tag variables that are available in semantic action.


    const char *t1, *t2;


    int t3, t4;


    // Autogenerated tag variables used by the lexer to track tag values.


    /*!stags:re2c format = 'const char *@@ = NULL;'; */


    /*!mtags:re2c format = 'int @@ = MTAG_ROOT;'; */


    /*!re2c


        re2c:api:style = free-form;


        re2c:define:YYCTYPE = char;


        re2c:define:YYSTAGP = "@@ = YYCURSOR;";


        re2c:define:YYSTAGN = "@@ = NULL;";


        re2c:define:YYMTAGP = "add_mtag(mt, @@, YYCURSOR);";


        re2c:define:YYMTAGN = "add_mtag(mt, @@, NULL);";


        re2c:yyfill:enable = 0;


        re2c:tags = 1;


        num = [0-9]+;


        @t1 num @t2 ("." #t3 num #t4)* [\x00] {


            ver.clear();


            ver.push_back(s2n(t1, t2));


            unfold(mt, t3, t4, ver);


            return true;


        }


        * { return false; }


    */
}
int main() {


    Ver v;


    assert(parse("1", v) && v == Ver({1}));


    assert(parse("1.2.3.4.5.6.7", v) && v == Ver({1, 2, 3, 4, 5, 6, 7}));


    assert(!parse("1.2.", v));


    return 0;
}

// re2c $INPUT -o $OUTPUT -8 --case-ranges -i
#include <assert.h>
#include <stdint.h>
/*!include:re2c "unicode_categories.re" */
static int lex(const char *s) {


    const char *YYCURSOR = s, *YYMARKER;


    /*!re2c


        re2c:define:YYCTYPE = 'unsigned char';


        re2c:yyfill:enable = 0;


        // Simplified "Unicode Identifier and Pattern Syntax"


        // (see https://unicode.org/reports/tr31)


        id_start    = L | Nl | [$_];


        id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3];


        identifier  = id_start id_continue*;


        identifier { return 0; }


        *          { return 1; }


    */
}
int main() {


    assert(lex("_Ыдентификатор") == 0);


    return 0;
}

typedef enum { OK, FAIL } Result;
/*!re2c


    number = [1-9][0-9]*;
*/

// floating-point numbers
frac  = [0-9]* "." [0-9]+ | [0-9]+ ".";
exp   = 'e' [+-]? [0-9]+;
float = frac exp? | [0-9]+ exp;
float { return OK; }

// re2c $INPUT -o $OUTPUT -i
#include <assert.h>
/*!include:re2c "definitions.h" */
Result lex(const char *s) {


    const char *YYCURSOR = s, *YYMARKER;


    /*!re2c


        re2c:define:YYCTYPE = char;


        re2c:yyfill:enable = 0;


        *      { return FAIL; }


        number { return OK; }


        !include "extra_rules.re.inc";


    */
}
int main() {


    assert(lex("123") == OK);


    assert(lex("123.4567") == OK);


    return 0;
}

// re2c $INPUT -o $OUTPUT -i --header lexer/state.h
#include <assert.h>
#include <stddef.h>
#include "lexer/state.h" // the header is generated by re2c
/*!header:re2c:on*/
typedef struct {


    const char *str, *cur;


    /*!stags:re2c format = "const char *@@;"; */
} LexerState;
/*!header:re2c:off*/
long lex(LexerState* st) {


    const char *t;


    /*!re2c


        re2c:header = "lexer/state.h";


        re2c:yyfill:enable = 0;


        re2c:define:YYCTYPE = char;


        re2c:define:YYCURSOR = "st->cur";


        re2c:tags = 1;


        re2c:tags:expression = "st->@@";


        [a]* @t [b]* { return t - st->str; }


    */
}
int main() {


    const char *s = "ab";


    LexerState st = { s, s /*!stags:re2c format = ", NULL"; */ };


    assert(lex(&st) == 1);


    return 0;
}

/* Generated by re2c */
typedef struct {


    const char *str, *cur;


    const char *yyt1;
} LexerState;