code-template ::


      name '=' code-exprs ';'


    | CODE_TEMPLATE ';'


    | '<undefined>' ';'
code-exprs ::


      <EMPTY>


    | code-exprs code-expr
code-expr ::


      STRING


    | VARIABLE


    | optional


    | list
optional ::


      '(' CONDITIONAL '?' code-exprs ')'


    | '(' CONDITIONAL '?' code-exprs ':' code-exprs ')'
list ::


      '[' VARIABLE ':' code-exprs ']'


    | '[' VARIABLE '{' NUMBER '}' ':' code-exprs ']'


    | '[' VARIABLE '{' NUMBER ',' NUMBER '}' ':' code-exprs ']'

A code template is a sequence of string literals, variables, optional elements and lists, or a reference to another code template, or a special value <undefined>. Variables are placeholders that are substituted during code generation phase. List variables are special: when expanding list templates, re2d repeats expressions the right hand side of the column a few times, each time replacing occurrences of the list variable with a value specific to this repetition. Lists have optional bounds (negative values are counted from the end, e.g. -1 means the last element). Conditional names start with a dot. Both conditionals and variables may be either local (specific to the given code template) or global (allowed in all code templates). When re2d reads syntax file, it checks that each code template uses only the variables and conditionals that are allowed in it.

For example, the following code template defines if-then-else construct for a C-like language:

code:if_then_else =


    [branch{0}: topindent "if " cond " {" nl


        indent [stmt: stmt] dedent]


    [branch{1:-1}: topindent "} else" (.cond ? " if " cond) " {" nl


        indent [stmt: stmt] dedent]


    topindent "}" nl;

Here branch is a list variable: branch{0} expands to the first branch (which is special, as there is no else part), branch{1:-1} expands to all remaining branches (if any). stmt is also a list variable: [stmt: stmt] is a nested list that expands to a list of statements in the body of the current branch. topindent, indent, dedent and nl are global variables, and .cond is a local conditional (their meaning is described below). This code template could produce the following code:

if x {


    // do something
} else if y {


    // do something else
} else {


    // don't do anything
}

Here's a list of all code templates supported by re2d with their local variables and conditionals. Note that a particular definition may, but does not have to use local variables and conditionals. Any unused code templates should be set to <undefined>.

Here's a list of all global variables that are allowed in syntax files:

Here's a list of all global conditionals that are allowed in syntax files:

// re2d $INPUT -o $OUTPUT
module main;
// Expect a null-terminated string.
private int lex(const(char)* yycursor) {


    uint count = 0;


    for (;;) {


    /*!re2c


        re2c:YYCTYPE = char;


        re2c:yyfill:enable = 0;


        *      { return -1; }


        [\x00] { return count; }


        [a-z]+ { ++count; continue; }


        [ ]+   { continue; }


    */


    }


    assert(0); // unreachable
}
void main() {


    assert(lex("") == 0);


    assert(lex("one two three") == 3);


    assert(lex("f0ur") == -1);
}

// re2d $INPUT -o $OUTPUT
module main;
// Expect a null-terminated string.
private int lex(immutable char[] s) {


    const(char)* yycursor = s.ptr, yylimit = s.ptr + s.length, yymarker;


    int count = 0;


    for (;;) {


    /*!re2c


        re2c:YYCTYPE = char;


        re2c:yyfill:enable = 0;


        re2c:eof = 0;


        str = ['] ([^'\\] | [\\][^])* ['];


        *    { return -1; }


        $    { return count; }


        str  { ++count; continue; }


        [ ]+ { continue; }


    */


    }


    assert(0); // unreachable
}
void main() {


    assert(lex("") == 0);


    assert(lex("'qu\0tes' 'are' 'fine: \\'' ") == 3);


    assert(lex("'unterminated\\'") == -1);
}

// re2d $INPUT -o $OUTPUT
module main;
import core.stdc.stdlib;
import core.stdc.string;
/*!max:re2c*/
private int lex(immutable char[] s) {


    // Make a copy of the string with YYMAXFILL zeroes at the end.


    char *buf = cast(char*) malloc(s.length + YYMaxFill);


    memcpy(buf, cast(const(void*)) s, s.length);


    memset(buf + s.length, 0, YYMaxFill);


    const(char)* yycursor = buf;


    const(char)* yylimit = buf + s.length + YYMaxFill;


    int count = 0;
loop:


    /*!re2c


        re2c:YYCTYPE = char;


        re2c:YYFILL = "goto fail;";


        str = ['] ([^'\\] | [\\][^])* ['];


        [\x00] {


            // Check that it is the sentinel, not some unexpected null.


            if (yycursor - 1 == buf + s.length) goto exit; else goto fail;


        }


        str  { ++count; goto loop; }


        [ ]+ { goto loop; }


        *    { goto fail; }


    */
fail:


    count = -1;
exit:


    free(buf);


    return count;
}
void main() {


    assert(lex("") == 0);


    assert(lex("'qu\0tes' 'are' 'fine: \\'' ") == 3);


    assert(lex("'unterminated\\'") == -1);


    assert(lex("'unexpected \0 null\\'") == -1);
}

// re2d $INPUT -o $OUTPUT
module main;
import core.stdc.stdlib;
import core.stdc.string;
private int lex(immutable char[] s) {


    // For the sake of example create a string without terminating null.


    char *buf = cast(char*) malloc(s.length);


    memcpy(buf, cast(const(void*)) s, s.length);


    const(char) *cur = buf, lim = buf + s.length;


    int count = 0;


    for (;;) {


    /*!re2c


        re2c:api = generic;


        re2c:yyfill:enable = 0;


        re2c:YYCTYPE = char;


        re2c:YYPEEK = "cur < lim ? *cur : 0";  // fake null


        re2c:YYSKIP = "++cur;";


        *      { count = -1; break; }


        [\x00] { break;{} }


        [a-z]+ { ++count; continue;{} }


        [ ]+   { continue; }


    */


    }


    free(buf);


    return count;
}
void main() {


    assert(lex("") == 0);


    assert(lex("one two three ") == 3);


    assert(lex("f0ur") == -1);
}

               <-- shift -->


             >-A------------B---------C-------------D#-----------E->


             buffer       token    marker         limit,


                                                  cursor
>-A------------B---------C-------------D------------E#->


             buffer,  marker        cursor        limit


             token


               <-- shift -->


             >-A------------B---------C-------------D#--E (EOF)


             buffer       token    marker         limit,


                                                  cursor
>-A------------B---------C-------------D---E#........


             buffer,  marker       cursor limit


             token

// re2d $INPUT -o $OUTPUT
module main;
import core.stdc.string;
import core.stdc.stdio;
enum BUFSIZE = 4095;
struct Input {


    FILE* file;


    char[BUFSIZE + 1] buffer;// +1 for sentinel


    char* yylimit, yycursor, yymarker, token;


    bool eof;
};
private int fill(ref Input it) {


    if (it.eof) return 1;


    const size_t shift = it.token - it.buffer.ptr;


    const size_t used = it.yylimit - it.token;


    // Error: lexeme too long. In real life could reallocate a larger buffer.


    if (shift < 1) return 2;


    // Shift buffer contents (discard everything up to the current token).


    memmove(cast(void*)it.buffer.ptr, it.token, used);


    it.yylimit -= shift;


    it.yycursor -= shift;


    it.yymarker -= shift;


    it.token -= shift;


    // Fill free space at the end of buffer with new data from file.


    it.yylimit += fread(it.yylimit, 1, BUFSIZE - used, it.file);


    it.yylimit[0] = 0;


    it.eof = it.yylimit < (it.buffer.ptr + BUFSIZE);


    return 0;
}
private int lex(ref Input yyrecord) {


    int count = 0;


    for (;;) {


        yyrecord.token = yyrecord.yycursor;


    /*!re2c


        re2c:api = record;


        re2c:YYCTYPE = "char";


        re2c:YYFILL = "fill(yyrecord) == 0";


        re2c:eof = 0;


        str = ['] ([^'\\] | [\\][^])* ['];


        *    { return -1; }


        $    { return count; }


        str  { ++count; continue; }


        [ ]+ { continue; }


    */


    }


    assert(0);
}
void main() {


    const char[] fname = "input";


    const char[] content = "'qu\0tes' 'are' 'fine: \\'' ";


    // Prepare input file: a few times the size of the buffer, containing


    // strings with zeroes and escaped quotes.


    FILE* f = fopen(fname.ptr, "w");


    for (int i = 0; i < BUFSIZE; ++i) {


        fwrite(cast(const(void*)) content.ptr, 1, content.length - 1, f);


    }


    fclose(f);


    int count = 3 * BUFSIZE; // number of quoted strings written to file


    // Initialize lexer state: all pointers are at the end of buffer.


    Input it;


    it.file = fopen(fname.ptr, "r");


    it.yycursor = it.yymarker = it.token = it.yylimit = it.buffer.ptr + BUFSIZE;


    it.eof = 0;


    // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL.


    it.yylimit[0] = 0;


    // Run the lexer.


    assert(lex(it) == count);


    // Cleanup: remove input file.


    fclose(it.file);


    remove(fname.ptr);
}

               <-- shift -->                 <-- need -->


             >-A------------B---------C-----D-------E---F--------G->


             buffer       token    marker cursor  limit
>-A------------B---------C-----D-------E---F--------G->


             buffer,  marker cursor               limit


             token


               <-- shift -->                 <-- need -->


             >-A------------B---------C-----D-------E-F        (EOF)


             buffer       token    marker cursor  limit
>-A------------B---------C-----D-------E-F###############


             buffer,  marker cursor                   limit


             token                        <- YYMAXFILL ->

// re2d $INPUT -o $OUTPUT
module main;
import core.stdc.string;
import core.stdc.stdio;
/*!max:re2c*/
enum BufSize = (4096 - YYMaxFill);
struct Input {


    FILE* file;


    char[BufSize + YYMaxFill] buffer;


    char* yylimit, yycursor, token;


    bool eof;
};
private int fill(ref Input it, size_t need) {


    if (it.eof) return 1;


    const size_t shift = it.token - it.buffer.ptr;


    const size_t used = it.yylimit - it.token;


    // Error: lexeme too long. In real life could reallocate a larger buffer.


    if (shift < need) return 2;


    // Shift buffer contents (discard everything up to the current token).


    memmove(it.buffer.ptr, it.token, used);


    it.yylimit -= shift;


    it.yycursor -= shift;


    it.token -= shift;


    // Fill free space at the end of buffer with new data from file.


    it.yylimit += fread(it.yylimit, 1, BufSize - used, it.file);


    // If read less than expected, this is end of input => add zero padding


    // so that the lexer can access characters at the end of buffer.


    if (it.yylimit < it.buffer.ptr + BufSize) {


        it.eof = true;


        memset(it.yylimit, 0, YYMaxFill);


        it.yylimit += YYMaxFill;


    }


    return 0;
}
private int lex(ref Input yyrecord) {


    int count = 0;


    for (;;) {


        yyrecord.token = yyrecord.yycursor;


    /*!re2c


        re2c:api = record;


        re2c:YYCTYPE = "char";


        re2c:YYFILL = "if (fill(yyrecord, @@) != 0) return -1;";


        str = ['] ([^'\\] | [\\][^])* ['];


        [\x00] {


            // Check that it is the sentinel, not some unexpected null.


            return yyrecord.token == yyrecord.yylimit - YYMaxFill ? count : -1;


        }


        str  { ++count; continue; }


        [ ]+ { continue; }


        *    { return -1; }


    */


    }


    assert(0);
}
void main() {


    const char[] fname = "input";


    const char[] content = "'qu\0tes' 'are' 'fine: \\'' ";


    // Prepare input file: a few times the size of the buffer, containing


    // strings with zeroes and escaped quotes.


    FILE* f = fopen(fname.ptr, "w");


    for (int i = 0; i < BufSize; ++i) {


        fwrite(content.ptr, 1, content.length - 1, f);


    }


    fclose(f);


    int count = 3 * BufSize; // number of quoted strings written to file


    // Initialize lexer state: all pointers are at the end of buffer.


    // This immediately triggers YYFILL, as the check `it.yycursor < it.yylimit` fails.


    Input it;


    it.file = fopen(fname.ptr, "r");


    it.yycursor = it.token = it.yylimit = it.buffer.ptr + BufSize;


    it.eof = 0;


    // Run the lexer.


    assert(lex(it) == count);


    // Cleanup: remove input file.


    fclose(it.file);


    remove(fname.ptr);
}

// re2d $INPUT -o $OUTPUT -i
module main;
enum ERROR = ulong.max;
private void add(ulong BASE)(ref ulong u, int d) {


    u = u * BASE + d;


    if (u > uint.max) { u = ERROR; }
}
private ulong parse_u32(const(char)* s) {


    const(char)* yycursor = s, yymarker;


    ulong u = 0;


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:YYCTYPE = char;


        end = "\x00";


        '0b' / [01]        { goto bin; }


        "0"                { goto oct; }


        "" / [1-9]         { goto dec; }


        '0x' / [0-9a-fA-F] { goto hex; }


        *                  { return ERROR; }


    */
bin:


    /*!re2c


        end   { return u; }


        [01]  { add!(2)(u, yycursor[-1] - '0'); goto bin; }


        *     { return ERROR; }


    */
oct:


    /*!re2c


        end   { return u; }


        [0-7] { add!(8)(u, yycursor[-1] - '0'); goto oct; }


        *     { return ERROR; }


    */
dec:


    /*!re2c


        end   { return u; }


        [0-9] { add!(10)(u, yycursor[-1] - '0'); goto dec; }


        *     { return ERROR; }


    */
hex:


    /*!re2c


        end   { return u; }


        [0-9] { add!(16)(u, yycursor[-1] - '0');      goto hex; }


        [a-f] { add!(16)(u, yycursor[-1] - 'a' + 10); goto hex; }


        [A-F] { add!(16)(u, yycursor[-1] - 'A' + 10); goto hex; }


        *     { return ERROR; }


    */
}
void main() {


    assert(parse_u32("") == ERROR);


    assert(parse_u32("1234567890") == 1234567890);


    assert(parse_u32("0b1101") == 13);


    assert(parse_u32("0x7Fe") == 2046);


    assert(parse_u32("0644") == 420);


    assert(parse_u32("9999999999") == ERROR);
}

// re2d $INPUT -o $OUTPUT -ci
module main;
enum ERROR = ulong.max;
/*!conditions:re2c*/
private void add(ulong BASE)(ref ulong u, int d) {


    u = u * BASE + d;


    if (u > uint.max) { u = ERROR; }
}
private ulong parse_u32(const(char)* s) {


    const(char)* yycursor = s, yymarker;


    YYCond yycond = YYCond.yycinit;


    ulong u = 0;


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:YYCTYPE = char;


        <*> * { return ERROR; }


        <init> '0b' / [01]        :=> bin


        <init> "0"                :=> oct


        <init> "" / [1-9]         :=> dec


        <init> '0x' / [0-9a-fA-F] :=> hex


        <bin, oct, dec, hex> "\x00" { return u; }


        <bin> [01]  { add!(2)(u,  yycursor[-1] - '0');      goto yyc_bin; }


        <oct> [0-7] { add!(8)(u,  yycursor[-1] - '0');      goto yyc_oct; }


        <dec> [0-9] { add!(10)(u, yycursor[-1] - '0');      goto yyc_dec; }


        <hex> [0-9] { add!(16)(u, yycursor[-1] - '0');      goto yyc_hex; }


        <hex> [a-f] { add!(16)(u, yycursor[-1] - 'a' + 10); goto yyc_hex; }


        <hex> [A-F] { add!(16)(u, yycursor[-1] - 'A' + 10); goto yyc_hex; }


    */
}
void main() {


    assert(parse_u32("") == ERROR);


    assert(parse_u32("1234567890") == 1234567890);


    assert(parse_u32("0b1101") == 13);


    assert(parse_u32("0x7Fe") == 2046);


    assert(parse_u32("0644") == 420);


    assert(parse_u32("9999999999") == ERROR);
}

// re2d $INPUT -o $OUTPUT -f
module main;
import core.stdc.stdio;
import core.stdc.string;
// Use a small buffer to cover the case when a lexeme doesn't fit.
// In real world use a larger buffer.
enum BUFSIZE = 10;
struct State {


    FILE* file;


    char[BUFSIZE + 1] buffer;


    char* yylimit, yycursor, yymarker, token;


    int yystate;
};
enum Status {END, READY, WAITING, BAD_PACKET, BIG_PACKET};
private Status fill(ref State st) {


    const size_t shift = st.token - cast(char*)st.buffer;


    const size_t used = st.yylimit - st.token;


    const size_t free = BUFSIZE - used;


    // Error: no space. In real life can reallocate a larger buffer.


    if (free < 1) return Status.BIG_PACKET;


    // Shift buffer contents (discard already processed data).


    memmove(cast(void*)st.buffer, st.token, used);


    st.yylimit -= shift;


    st.yycursor -= shift;


    st.yymarker -= shift;


    st.token -= shift;


    // Fill free space at the end of buffer with new data.


    const size_t read = fread(st.yylimit, 1, free, st.file);


    st.yylimit += read;


    st.yylimit[0] = 0; // append sentinel symbol


    return Status.READY;
}
private Status lex(ref State yyrecord, uint* recv) {


    char yych;


    /*!getstate:re2c*/


    for (;;) {


        yyrecord.token = yyrecord.yycursor;


    /*!re2c


        re2c:api = record;


        re2c:YYCTYPE = char;


        re2c:YYFILL = "return Status.WAITING;";


        re2c:eof = 0;


        packet = [a-z]+[;];


        *      { return Status.BAD_PACKET; }


        $      { return Status.END; }


        packet { *recv = *recv + 1; continue; }


    */


    }


    assert(0); // unreachable
}
private void test(string[] packets, Status expect) {


    // Create a pipe (open the same file for reading and writing).


    const(char*) fname = "pipe";


    FILE* fw = fopen(fname, "w");


    FILE* fr = fopen(fname, "r");


    setvbuf(fw, null, _IONBF, 0);


    setvbuf(fr, null, _IONBF, 0);


    // Initialize lexer state: `state` value is -1, all pointers are at the end


    // of buffer.


    State st;


    st.file = fr;


    st.yycursor = st.yymarker = st.token = st.yylimit = cast(char*)st.buffer + BUFSIZE;


    // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL.


    st.yylimit[0] = 0;


    st.yystate = -1;


    // Main loop. The buffer contains incomplete data which appears packet by


    // packet. When the lexer needs more input it saves its internal state and


    // returns to the caller which should provide more input and resume lexing.


    Status status;


    uint send = 0, recv = 0;


    for (;;) {


        status = lex(st, &recv);


        if (status == Status.END) {


            debug{printf("done: got %u packets\n", recv);}


            break;


        } else if (status == Status.WAITING) {


            debug{printf("waiting...\n");}


            if (send < packets.length) {


                debug{printf("sent packet %u\n", send);}


                fprintf(fw, "%s", cast(char*)packets[send]);


                ++send;


            }


            status = fill(st);


            debug{printf("queue: '%s'\n", cast(char*)st.buffer);}


            if (status == Status.BIG_PACKET) {


                debug{printf("error: packet too big\n");}


                break;


            }


            assert(status == Status.READY);


        } else {


            assert(status == Status.BAD_PACKET);


            debug{printf("error: ill-formed packet\n");}


            break;


        }


    }


    // Check results.


    assert(status == expect);


    if (status == Status.END) assert(recv == send);


    // Cleanup: remove input file.


    fclose(fw);


    fclose(fr);


    remove(fname);
}
void main() {


    string[] packets1 = [];


    string[] packets2 = ["zero;", "one;", "two;", "three;", "four;"];


    string[] packets3 = ["zer0;"];


    string[] packets4 = ["looooooooooong;"];


    test(packets1, Status.END);


    test(packets2, Status.END);


    test(packets3, Status.BAD_PACKET);


    test(packets4, Status.BIG_PACKET);
}

// re2d $INPUT -o $OUTPUT
module main;
// This example shows how to combine reusable re2c blocks: two blocks
// ('colors' and 'fish') are merged into one. The 'salmon' rule occurs
// in both blocks; the 'fish' block takes priority because it is used
// earlier. Default rule * occurs in all three blocks; the local (not
// inherited) definition takes priority.
enum What { COLOR, FISH, DUNNO };
/*!rules:re2c:colors


    *                            { assert(false); }


    "red" | "salmon" | "magenta" { return What.COLOR; }
*/
/*!rules:re2c:fish


    *                            { assert(false); }


    "haddock" | "salmon" | "eel" { return What.FISH; }
*/
private What lex(const(char)* s) {


    const(char)* yycursor = s, yymarker;


    /*!re2c


        re2c:YYCTYPE = "char";


        re2c:yyfill:enable = 0;


        !use:fish;


        !use:colors;


        * { return What.DUNNO; } // overrides inherited '*' rules


    */
}
void main() {


    assert(lex("salmon") == What.FISH);


    assert(lex("what?") == What.DUNNO);
}

// re2d $INPUT -o $OUTPUT --input-encoding utf8
module main;
import std.stdint;
// This example supports multiple input encodings: UTF-8 and UTF-32.
// Both lexers are generated from the same rules block, and the use
// blocks add only encoding-specific configurations.
/*!rules:re2c


    re2c:yyfill:enable = 0;


    "∀x ∃y" { return 0; }


    *       { return 1; }
*/
private int lex_utf8(const(uint8_t)* s) {


    const(uint8_t)* yycursor = s, yymarker;


    /*!use:re2c


        re2c:YYCTYPE = uint8_t;


        re2c:encoding:utf8 = 1;


    */
}
private int lex_utf32(const(uint32_t)* s) {


    const(uint32_t)* yycursor = s, yymarker;


    /*!use:re2c


        re2c:YYCTYPE = uint32_t;


        re2c:encoding:utf32 = 1;


    */
}
void main() {


    immutable uint8_t[] s8 = // UTF-8


        [ 0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79 ];


    immutable uint32_t[] s32 = // UTF32


        [ 0x00002200, 0x00000078, 0x00000020, 0x00002203, 0x00000079 ];


    assert(lex_utf8(cast(const(uint8_t)*)s8) == 0);


    assert(lex_utf32(cast(const(uint32_t)*)s32) == 0);
}

// re2d $INPUT -o $OUTPUT
module main;
struct SemVer {


    int major;


    int minor;


    int patch;
};
private int s2n(const(char)* s, const(char)* e) { // pre-parsed string to number


    int n = 0;


    for (; s < e; ++s) n = n * 10 + (*s - '0');


    return n;
}
private bool lex(const(char)* str, ref SemVer ver) {


    const(char)* yycursor = str, yymarker;


    // Final tag variables available in semantic action.


    /*!svars:re2c format = 'const(char)* @@;\n'; */


    // Intermediate tag variables used by the lexer (must be autogenerated).


    /*!stags:re2c format = 'const(char)* @@;\n'; */


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:tags = 1;


        re2c:YYCTYPE = "char";


        num = [0-9]+;


        @t1 num @t2 "." @t3 num @t4 ("." @t5 num)? [\x00] {


            ver.major = s2n(t1, t2);


            ver.minor = s2n(t3, t4);


            ver.patch = t5 != null ? s2n(t5, yycursor - 1) : 0;


            return true;


        }


        * { return false; }


    */
}
void main() {


    SemVer v;


    assert(lex("23.34", v) && v.major == 23 && v.minor == 34 && v.patch == 0);


    assert(lex("1.2.999", v) && v.major == 1 && v.minor == 2 && v.patch == 999);


    assert(!lex("1.a", v));
}

// re2d $INPUT -o $OUTPUT --tags
module main;
import core.stdc.string;
import core.stdc.stdio;
import std.stdio;
enum BUFSIZE = 4095;
struct Input {


    FILE* file;


    char[BUFSIZE + 1] buffer;// +1 for sentinel


    char* yylimit, yycursor, yymarker, token;


    // Intermediate tag variables must be part of the lexer state passed to YYFILL.


    // They don't correspond to tags and should be autogenerated by re2c.


    /*!stags:re2c format = 'char* @@;'; */


    bool eof;
};
struct SemVer {


    int major;


    int minor;


    int patch;
};
private int s2n(const(char)* s, const(char)* e) { // pre-parsed string to number


    int n = 0;


    for (; s < e; ++s) n = n * 10 + (*s - '0');


    return n;
}
private int fill(ref Input it) {


    if (it.eof) return 1;


    const size_t shift = it.token - it.buffer.ptr;


    const size_t used = it.yylimit - it.token;


    // Error: lexeme too long. In real life could reallocate a larger buffer.


    if (shift < 1) return 2;


    // Shift buffer contents (discard everything up to the current token).


    memmove(cast(void*)it.buffer.ptr, it.token, used);


    it.yylimit -= shift;


    it.yycursor -= shift;


    it.yymarker -= shift;


    it.token -= shift;


    // Tag variables need to be shifted like other input positions. The check


    // for non-null is only needed if some tags are nested inside of alternative


    // or repetition, so that they can have null value.


    /*!stags:re2c format = "if (it.@@) it.@@ -= shift;\n"; */


    // Fill free space at the end of buffer with new data from file.


    it.yylimit += fread(it.yylimit, 1, BUFSIZE - used, it.file);


    it.yylimit[0] = 0;


    it.eof = it.yylimit < (it.buffer.ptr + BUFSIZE);


    return 0;
}
private bool lex(ref Input yyrecord, ref SemVer[] vers) {


    // Final variables available in semantic actions.


    /*!svars:re2c format = 'char* @@;'; */


    for (;;) {


        yyrecord.token = yyrecord.yycursor;


    /*!re2c


        re2c:api = record;


        re2c:YYCTYPE = "char";


        re2c:YYFILL = "fill(yyrecord) == 0";


        re2c:eof = 0;


        num = [0-9]+;


        num @t1 "." @t2 num @t3 ("." @t4 num)? [\n] {


            int major = s2n(yyrecord.token, t1);


            int minor = s2n(t2, t3);


            int patch = t4 != null ? s2n(t4, yyrecord.yycursor - 1) : 0;


            SemVer ver = SemVer(major, minor, patch);


            vers ~= ver;


            continue;


        }


        $ { return true; }


        * { return false; }


    */


    }


    assert(0);
}
void main() {


    const char[] fname = "input";


    const char[] content = "1.22.333\n' ";


    SemVer[BUFSIZE] expect = SemVer(1, 22, 333);


    SemVer[] actual;


    // Prepare input file: a few times the size of the buffer, containing


    // strings with zeroes and escaped quotes.


    FILE* f = fopen(fname.ptr, "w");


    for (int i = 0; i < BUFSIZE; ++i) {


        fwrite(cast(const(void*)) content.ptr, 1, content.length - 2, f); // skip null-terminator


    }


    fclose(f);


    // Initialize lexer state: all pointers are at the end of buffer.


    Input it;


    it.file = fopen(fname.ptr, "r");


    it.yycursor = it.yymarker = it.token = it.yylimit = it.buffer.ptr + BUFSIZE;


    it.eof = 0;


    // Sentinel (at YYLIMIT pointer) is set to zero, which triggers YYFILL.


    it.yylimit[0] = 0;


    // Run the lexer.


    assert(lex(it, actual) && actual == expect);


    // Cleanup: remove input file.


    fclose(it.file);


    remove(fname.ptr);
}

// re2d $INPUT -o $OUTPUT
module main;
struct SemVer { int major, minor, patch; };
private int s2n(const(char)* s, const(char)* e) { // pre-parsed string to number


    int n = 0;


    for (; s < e; ++s) n = n * 10 + (*s - '0');


    return n;
}
private bool lex(const(char)* str, ref SemVer ver) {


    const(char)* yycursor = str, yymarker;


    // Final tag variables available in semantic action.


    /*!svars:re2c format = 'const(char)* @@;\n'; */


    // Intermediate tag variables used by the lexer (must be autogenerated).


    /*!stags:re2c format = 'const(char)* @@;\n'; */


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:captvars = 1;


        re2c:YYCTYPE = "char";


        num = [0-9]+;


        (num) "." (num) ("." num)? [\x00] {


            ver.major = s2n(yytl1, yytr1);


            ver.minor = s2n(yytl2, yytr2);


            ver.patch = yytl3 ? s2n(yytl3 + 1, yytr3) : 0;


            return true;


        }


        * { return false; }


    */
}
void main() {


    SemVer v;


    assert(lex("23.34", v) && v.major == 23 && v.minor == 34 && v.patch == 0);


    assert(lex("1.2.999", v) && v.major == 1 && v.minor == 2 && v.patch == 999);


    assert(!lex("1.a", v));
}

// re2d $INPUT -o $OUTPUT
module main;
enum MtagRoot = -1;
// An m-tag tree is a way to store histories with an O(1) copy operation.
// Histories naturally form a tree, as they have common start and fork at some
// point. The tree is stored as an array of pairs (tag value, link to parent).
// An m-tag is represented with a single link in the tree (array index).
struct Mtag {


    const(char)* elem; // tag value


    int pred; // index of the predecessor node or root
};
alias MtagTrie = Mtag[];
alias Ver = int[];
private int s2n(const(char)* s, const(char)* e) { // pre-parsed string to number


    int n = 0;


    for (; s < e; ++s) n = n * 10 + (*s - '0');


    return n;
}
// Append a single value to an m-tag history.
private void add_mtag(ref MtagTrie trie, ref int mtag, const(char)* value) {


    Mtag m = {value, mtag};


    mtag = cast(int)trie.length;


    trie ~= [m];
}
// Recursively unwind tag histories and collect version components.
private void unfold(const ref MtagTrie trie, int x, int y, ref Ver ver) {


    // Reached the root of the m-tag tree, stop recursion.


    if (x == MtagRoot && y == MtagRoot) return;


    // Unwind history further.


    unfold(trie, trie[x].pred, trie[y].pred, ver);


    // Get tag values. Tag histories must have equal length.


    assert(x != MtagRoot && y != MtagRoot);


    const(char)* ex = trie[x].elem, ey = trie[y].elem;


    if (ex != null && ey != null) {


        // Both tags are valid pointers, extract component.


        ver ~= [s2n(ex, ey)];


    } else {


        // Both tags are null (this corresponds to zero repetitions).


        assert(ex == null && ey == null);


    }
}
private bool parse(const(char)* str, ref Ver ver) {


    const(char)* yycursor = str, yymarker;


    MtagTrie mt;


    // Final tag variables available in semantic action.


    /*!svars:re2c format = "const(char)* @@;"; */


    /*!mvars:re2c format = "int @@;"; */


    // Intermediate tag variables used by the lexer (must be autogenerated).


    /*!stags:re2c format = "const(char)* @@ = null;"; */


    /*!mtags:re2c format = "int @@ = MtagRoot;"; */


    /*!re2c


        re2c:yyfill:enable = 0;


        re2c:tags = 1;


        re2c:YYCTYPE = "char";


        re2c:YYMTAGP = "add_mtag(mt, @@, yycursor);";


        re2c:YYMTAGN = "add_mtag(mt, @@, null);";


        num = [0-9]+;


        @t1 num @t2 ("." #t3 num #t4)* [\x00] {


            ver = [];


            ver ~= [s2n(t1, t2)];


            unfold(mt, t3, t4, ver);


            return true;


        }


        * { return false; }


    */
}
void main() {


    Ver v;


    assert(parse("1", v) && v == [1]);


    assert(parse("1.2.3.4.5.6.7", v) && v == [1, 2, 3, 4, 5, 6, 7]);


    assert(!parse("1.2.", v));
}

// re2d $INPUT -o $OUTPUT -8 -i
module main;
/*!include:re2c "unicode_categories.re" */
private int lex(const(char)* s) {


    const(char)* yycursor = s, yymarker;


    /*!re2c


        re2c:YYCTYPE = "char";


        re2c:yyfill:enable = 0;


        // Simplified "Unicode Identifier and Pattern Syntax"


        // (see https://unicode.org/reports/tr31)


        id_start    = L | Nl | [$_];


        id_continue = id_start | Mn | Mc | Nd | Pc | [\u200D\u05F3];


        identifier  = id_start id_continue*;


        identifier { return 0; }


        *          { return 1; }


    */
}
void main() {


    assert(lex("_Ыдентификатор") == 0);


    assert(lex("!!!")==1);
}

enum Result{ OK, FAIL };
/*!re2c


    number = [1-9][0-9]*;
*/

// floating-point numbers
frac  = [0-9]* "." [0-9]+ | [0-9]+ ".";
exp   = 'e' [+-]? [0-9]+;
float = frac exp? | [0-9]+ exp;
float { return Result.OK; }

// re2d $INPUT -o $OUTPUT -i
/*!include:re2c "definitions.d" */
private Result lex(const(char)* s) {


    const(char)* yycursor = s, yymarker;


    /*!re2c


        re2c:YYCTYPE = "char";


        re2c:yyfill:enable = 0;


        *      { return Result.FAIL; }


        number { return Result.OK; }


        !include "extra_rules.re.inc";


    */
}
void main() {


    assert(lex("123") == Result.OK);


    assert(lex("123.4567") == Result.OK);
}

// re2d $INPUT -o $OUTPUT -i --header lexer/state.d
module main;
import core.stdc.stddef;
import lexer.state; // the module is generated by re2c
/*!header:re2c:on*/
module lexer.state;
struct LexerState {


    const(char)* str, yycursor;


    /*!stags:re2c format = "const(char)* @@;"; */
};
/*!header:re2c:off*/
private long lex(ref LexerState yyrecord) {


    const(char)* t;


    /*!re2c


        re2c:api = record;


        re2c:YYCTYPE = "char";


        re2c:tags = 1;


        re2c:yyfill:enable = 0;


        re2c:header = "lexer/state.d";


        [a]* @t [b]* { return t - yyrecord.str; }


    */
}
void main() {


    const(char)* s = "ab";


    LexerState st = {s, s /*!stags:re2c format = ", null"; */};


    assert(lex(st) == 1);


    const(char)* s2 = "aaabbbbbbbb";


    LexerState st2 = {s2, s2 /*!stags:re2c format = ", null"; */};


    assert(lex(st2) == 3);
}

/* Generated by re2c */
module lexer.state;
struct LexerState {


    const(char)* str, yycursor;


    const(char)* yyt1;
};