src/MulleScionParser+Parsing.m
2995f4ad
 //
 //  MulleScionParser+Parsing.m
d2f30cd2
 //  MulleScion
2995f4ad
 //
 //  Created by Nat! on 26.02.13.
 //
ef82839d
 //  Copyright (c) 2013 Nat! - Mulle kybernetiK
 //  All rights reserved.
 //
 //  Redistribution and use in source and binary forms, with or without
 //  modification, are permitted provided that the following conditions are met:
 //
 //  Redistributions of source code must retain the above copyright notice, this
 //  list of conditions and the following disclaimer.
 //
 //  Redistributions in binary form must reproduce the above copyright notice,
 //  this list of conditions and the following disclaimer in the documentation
 //  and/or other materials provided with the distribution.
 //
 //  Neither the name of Mulle kybernetiK nor the names of its contributors
 //  may be used to endorse or promote products derived from this software
 //  without specific prior written permission.
 //
 //  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 //  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 //  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 //  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 //  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 //  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 //  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 //  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 //  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 //  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 //  POSSIBILITY OF SUCH DAMAGE.
 //
 
2995f4ad
 #import "MulleScionParser+Parsing.h"
 
801f8317
 #define MULLE_SCION_OBJECT_NEXT_POINTER_VISIBILITY  @public
 
 #import "MulleScionObjectModel+Parsing.h"
08b5d36d
 #import "MulleScionObjectModel+NSCoding.h"
 #import "MulleScionObjectModel+MacroExpansion.h"
b8c659f4
 #import "MulleScionObjectModel+TraceDescription.h"
 #if ! TARGET_OS_IPHONE
3f8d4c56
 # import <Foundation/NSDebug.h>
 #endif
2995f4ad
 
27dc2200
 @implementation MulleScionParser( Parsing)
2995f4ad
 
bdb5913e
 
 - (void) parserWarning:(parser_warning_info *) info
 {
    [self parser:info->parser
 warningInFileName:info->fileName ? info->fileName : @"template"
      lineNumber:info->lineNumber
          reason:info->message];
 }
 
 
 - (void) parserError:(parser_error_info *) info
 {
    [self parser:info->parser
 errorInFileName:info->fileName ? info->fileName : @"template"
      lineNumber:info->lineNumber
          reason:info->message];
 }
 
2995f4ad
 // parse it a little like Twig
 // {# comment #}
 // {{ identifier }}
 // {{ #number }}
 // {{ "string" }}
 // {{ [ identifier string expr ... ] }}
 // {% variable = expr %}
 // {% if expr %}
 // {% else %}
 // {% endif %}
 // {% for identifier in expr %}
 // {% endfor %}
 // {% while expr %}
 // {% endwhile %}
 // {% block identifier %}
 // {% endblock %}
 // {% includes "string" %}
 // {% extends "string" %}  // with extends, the loaded template is searched for
 //                         // blocks these are then substitued for the blocks
 //                         // of the same name, that already exist
 //
 typedef enum
 {
    eof        = -1,
    expression = 0,
    command    = 1,
04373918
    comment    = 2,
    garbage    = 3
2995f4ad
 } macro_type;
 
 
 typedef struct _parser_memo
 {
    unsigned char   *curr;
    NSUInteger      lineNumber;
 } parser_memo;
 
 
 typedef struct _parser
 {
08b5d36d
    unsigned char        *buf;
    unsigned char        *sentinel;
27dc2200
 
08b5d36d
    unsigned char        *curr;
    NSUInteger           lineNumber;
27dc2200
 
08b5d36d
    parser_memo          memo;
801f8317
    parser_memo          memo_scion;
c533990f
    parser_memo          memo_interesting;
27dc2200
 
08b5d36d
    MulleScionObject     *first;
070695ad
 
bdb5913e
    void                 (*parser_do_error)( id self, SEL sel, parser_warning_info *parser);
    void                 (*parser_do_warning)( id self, SEL sel, parser_error_info *parser);
08b5d36d
    id                   self;
    SEL                  sel;
776920e7
    int                  skipComments;
08b5d36d
    int                  inMacro;
    int                  wasMacroCall;
b8c659f4
    unsigned int         environment;
2995f4ad
    NSString             *fileName;
fec98eee
    MulleScionParserTables  tables;
    NSMutableArray       *converterStack;
2995f4ad
 } parser;
 
 
b8c659f4
 enum
 {
f958902e
    MULLESCION_ALLOW_GETENV_INCLUDES     = 0x01,
b8c659f4
    MULLESCION_NO_HASHBANG               = 0x02,
f958902e
    MULLESCION_VERBATIM_INCLUDE_HASHBANG = 0x04,
b8c659f4
    MULLESCION_DUMP_COMMANDS             = 0x08,
dc48b1ee
    MULLESCION_DUMP_EXPRESSIONS          = 0x10,
f958902e
    MULLESCION_DUMP_FILE_INCLUDES        = 0x20,
    MULLESCION_DUMP_MACROS               = 0x40
b8c659f4
 };
 
2523ba40
 static void   parser_skip_after_newline( parser *p);
 
 static void   parser_skip_initial_hashbang_line_if_present( parser *p)
 {
    if( p->sentinel - p->buf < 4)
       return;
 
a50e3201
    assert( p->lineNumber == 1);
2523ba40
    if( memcmp( "#!", p->buf, 2))
       return;
27dc2200
 
2523ba40
    // so assume mulle-scion was started as unix shellscrip
    parser_skip_after_newline( p);
 }
 
 
f958902e
 static int   getenv_yes_no_default( char *name, int default_value)
 {
    char   *s;
 
    s = getenv( name);
    if( ! s)
       return( default_value);
 
    switch( *s)
    {
    case 'f' :
    case 'F' :
    case 'n' :
    case 'N' :
    case '0' : return( 0);
    }
 
    return( 1);
 }
 
 
 static inline int  getenv_yes_no( char *name)
 {
    return( getenv_yes_no_default( name, 0));
 }
 
 
2995f4ad
 static void   parser_init( parser *p, unsigned char *buf, size_t len)
 {
    memset( p, 0, sizeof( parser));
    if( buf && len)
    {
       p->buf        = p->curr = buf;
       p->sentinel   = &p->buf[ len];
       p->lineNumber = 1;
    }
b8c659f4
 
f958902e
    p->environment |= getenv_yes_no( "MULLESCION_ALLOW_GETENV_INCLUDES") ? MULLESCION_ALLOW_GETENV_INCLUDES : 0;
    p->environment |= getenv_yes_no( "MULLESCION_NO_HASHBANG") ? MULLESCION_NO_HASHBANG : 0;
    p->environment |= getenv_yes_no( "MULLESCION_VERBATIM_INCLUDE_HASHBANG") ? MULLESCION_VERBATIM_INCLUDE_HASHBANG : 0;
    p->environment |= getenv_yes_no( "MULLESCION_DUMP_COMMANDS") ? MULLESCION_DUMP_COMMANDS : 0;
    p->environment |= getenv_yes_no( "MULLESCION_DUMP_EXPRESSIONS") ? MULLESCION_DUMP_EXPRESSIONS : 0;
    p->environment |= getenv_yes_no( "MULLESCION_DUMP_FILE_INCLUDES") ? MULLESCION_DUMP_FILE_INCLUDES : 0;
    p->environment |= getenv_yes_no( "MULLESCION_DUMP_MACROS") ? MULLESCION_DUMP_MACROS : 0;
2995f4ad
 }
 
da962797
 
 static inline void   parser_set_warning_callback( parser *p, id self, SEL sel)
 {
bdb5913e
    p->self              = self;
    p->sel               = sel;
da962797
    p->parser_do_warning = (void *) [p->self methodForSelector:sel];
 }
 
 
801f8317
 static inline void   parser_set_error_callback( parser *p, id self, SEL sel)
2995f4ad
 {
bdb5913e
    p->self            = self;
    p->sel             = sel;
2995f4ad
    p->parser_do_error = (void *) [p->self methodForSelector:sel];
 }
 
 
801f8317
 static inline void   parser_set_blocks_table( parser *p, NSMutableDictionary *table)
08b5d36d
 {
    NSCParameterAssert( ! table || [table isKindOfClass:[NSMutableDictionary class]]);
27dc2200
 
fec98eee
    p->tables.blockTable = table;
08b5d36d
 }
 
 
801f8317
 static inline void   parser_set_definitions_table( parser *p, NSMutableDictionary *table)
2995f4ad
 {
    NSCParameterAssert( ! table || [table isKindOfClass:[NSMutableDictionary class]]);
27dc2200
 
fec98eee
    p->tables.definitionTable = table;
2995f4ad
 }
 
08b5d36d
 
801f8317
 static inline void   parser_set_dependency_table( parser *p, NSMutableDictionary *table)
 {
    NSCParameterAssert( ! table || [table isKindOfClass:[NSMutableDictionary class]]);
27dc2200
 
fec98eee
    p->tables.dependencyTable = table;
801f8317
 }
 
 
 static inline void   parser_set_macro_table( parser *p, NSMutableDictionary *table)
08b5d36d
 {
    NSCParameterAssert( ! table || [table isKindOfClass:[NSMutableDictionary class]]);
27dc2200
 
fec98eee
    p->tables.macroTable = table;
08b5d36d
 }
 
 
801f8317
 static inline void   parser_set_filename( parser *p, NSString *s)
2995f4ad
 {
    p->fileName = s;
 }
 
da962797
 
 static NSString   *parser_diagnostic_string( parser *p, NSString *reason)
2995f4ad
 {
c533990f
    NSString       *s;
    size_t         p_len;
    size_t         s_len;
    size_t         i;
    unsigned char  *prefix;
    unsigned char  *suffix;
27dc2200
 
da962797
    //
    // p->memo_scion.curr is about the start of the parsed object
    // p->curr is where the parsage failed, try to print something interesting
    // near the parse failure (totally heuristic), but not too much
    //
    p_len = p->curr - p->memo_interesting.curr;
    if( p_len > 32)
       p_len = 32;
    if( p_len < 12)
       p_len += 3;
27dc2200
 
da962797
    s_len  = p_len >= 6 ? 12 : 12 + 6 - p_len;
    prefix = &p->curr[ -p_len];
    suffix = &p->curr[ 1];
bdb5913e
 
    if( suffix > p->sentinel)
       suffix = p->sentinel;
27dc2200
 
da962797
    if( prefix < p->buf)
2995f4ad
    {
da962797
       prefix = p->buf;
       p_len  = p->curr - p->buf;
    }
27dc2200
 
da962797
    if( &suffix[ s_len] > p->sentinel)
bdb5913e
       s_len  = p->sentinel - suffix;
27dc2200
 
da962797
    // stop tail at linefeed
    for( i = 0; i < s_len; i++)
       if( suffix[ i] == '\r' || suffix[ i] == '\n' || suffix[ i] == ';' || suffix[ i] == '}' || suffix[ i] == '%')
          break;
    s_len = i;
c533990f
 
       // terminal escape sequences
 #if HAVE_TERMINAL
 #define RED   "\033[01;31m"
 #define NONE  "\033[00m"
 #else
 #define RED   ""
 #define NONE  ""
 #endif
27dc2200
 
da962797
    s = [NSString stringWithFormat:@"%.*s" RED "%c" NONE "%.*s", (int) p_len, prefix, *p->curr, (int) s_len, suffix];
    s = [s stringByReplacingOccurrencesOfString:@"\n"
                                     withString:@" "];
    s = [s stringByReplacingOccurrencesOfString:@"\r"
                                     withString:@""];
    s = [s stringByReplacingOccurrencesOfString:@"\t"
                                     withString:@" "];
    s = [s stringByReplacingOccurrencesOfString:@"\""
                                     withString:@"\\\""];
    s = [s stringByReplacingOccurrencesOfString:@"'"
                                     withString:@"\\'"];
27dc2200
 
da962797
    s = [NSString stringWithFormat:@"at '%c' near \"%@\", %@", *p->curr, s, reason];
    return( s);
 }
 
 
 static void  parser_warning( parser *p, char *c_format, ...)
 {
bdb5913e
    NSString              *reason;
    parser_warning_info   info;
    va_list               args;
27dc2200
 
bdb5913e
    if( ! p->parser_do_warning)
       return;
27dc2200
 
bdb5913e
    va_start( args, c_format);
1ae081f1
 #ifdef __MULLE_OBJC__
bdb5913e
    reason = [[[NSString alloc] initWithFormat:[NSString stringWithCString:c_format]
27dc2200
                                    varargList:args] autorelease];
bdb5913e
 #else
    reason = [[[NSString alloc] initWithFormat:[NSString stringWithCString:c_format]
                                     arguments:args] autorelease];
 #endif
    va_end( args);
27dc2200
 
bdb5913e
    info.parser     = p;
    info.lineNumber = p->memo.lineNumber;
    info.message    = parser_diagnostic_string( p, reason);
    info.fileName   = p->fileName;
da962797
 
bdb5913e
    (*p->parser_do_warning)( p->self, p->sel, &info);
da962797
 }
 
 
 //
 // there is no return, stuff just leaks and we abort
 //
 static void  MULLE_NO_RETURN  parser_error( parser *p, char *c_format, ...)
 {
bdb5913e
    NSString            *reason;
    parser_error_info   info;
    va_list             args;
27dc2200
 
bdb5913e
    if( ! p->parser_do_error)
       abort();
27dc2200
 
bdb5913e
    va_start( args, c_format);
1ae081f1
 #ifdef __MULLE_OBJC__
bdb5913e
    reason = [[[NSString alloc] initWithFormat:[NSString stringWithCString:c_format]
27dc2200
                                    varargList:args] autorelease];
bdb5913e
 #else
    reason = [[[NSString alloc] initWithFormat:[NSString stringWithCString:c_format]
                                     arguments:args] autorelease];
 #endif
    va_end( args);
da962797
 
bdb5913e
    info.parser     = p;
    info.lineNumber = p->memo.lineNumber;
    info.message    = parser_diagnostic_string( p, reason);
    info.fileName   = p->fileName;
27dc2200
 
bdb5913e
    (*p->parser_do_error)( p->self, p->sel, &info);
68da4c13
    abort();
2995f4ad
 }
 
bdb5913e
 
c533990f
 static unsigned char    *unescaped_string_if_needed( unsigned char *s,
                                                      NSUInteger len,
                                                      NSUInteger *result_len)
 {
    unsigned char   *memo;
    unsigned char   *unescaped;
    unsigned char   *src;
    unsigned char   *dst;
    unsigned char   *sentinel;
    unsigned char   c;
    int             escaped;
    size_t          copy_len;
27dc2200
 
c533990f
    assert( s);
    assert( len);
    assert( result_len);
27dc2200
 
c533990f
    unescaped = NULL;
    dst       = NULL;
    memo      = s;
    sentinel  = &s[ len];
2995f4ad
 
c533990f
    escaped   = 0;
27dc2200
 
c533990f
    for( src = s; src < sentinel; src++)
    {
       c = *src;
27dc2200
 
c533990f
       if( escaped)
       {
          escaped = 0;
          switch( c)
          {
             case 'a'  : c = '\a'; break;
             case 'b'  : c = '\b'; break;
             case 'f'  : c = '\f'; break;
             case 'n'  : c = '\n'; break;
             case 'r'  : c = '\r'; break;
             case 't'  : c = '\t'; break;
             case 'v'  : c = '\v'; break;
             case '\"' : c = '\"'; break;
             case '\'' : c = '\''; break;
             case '?'  : c = '?';  break;
             case '\\' : c = '\\'; break;
27dc2200
 
c533990f
             // can't do numeric codes yet
          }
          *dst++ = c;
          continue;
       }
27dc2200
 
c533990f
       if( c == '\\')
       {
          escaped = 1;
          if( ! unescaped)
          {
             unescaped = malloc( len);
             if( ! unescaped)
                [NSException raise:NSMallocException
                            format:@"can't allocate %ld bytes", (long) len];
 
ef903235
             copy_len = (size_t) (src - memo);
c533990f
             memcpy( unescaped, memo, copy_len);
             dst = &unescaped[ copy_len];
          }
          continue;
       }
27dc2200
 
c533990f
       if( dst)
          *dst++ = c;
    }
27dc2200
 
c533990f
    assert( ! escaped);  // malformed, but so what ?
27dc2200
 
c533990f
    *result_len = dst - unescaped;
27dc2200
 
c533990f
    // maybe a little bit pedantic ?
    if( unescaped)
7334c90a
    {
c533990f
       unescaped = realloc( unescaped, *result_len);
7334c90a
       if( ! unescaped)
          [NSException raise:NSMallocException
                      format:@"can't shrink down to %ld bytes", (long) *result_len];
    }
27dc2200
 
c533990f
    return( unescaped);
 }
27dc2200
 
99fb8c65
 # pragma mark -
27dc2200
 # pragma mark Tokenizing
99fb8c65
 
801f8317
 static inline void   parser_memorize( parser *p, parser_memo *memo)
2995f4ad
 {
    memo->curr       = p->curr;
    memo->lineNumber = p->lineNumber;
 }
 
 
801f8317
 static inline void   parser_recall( parser *p, parser_memo *memo)
2995f4ad
 {
    p->curr       = memo->curr;
    p->lineNumber = memo->lineNumber;
 }
 
776920e7
 
801f8317
 static inline void   parser_nl( parser *p)
 {
    p->lineNumber++;
 }
 
fc1b9d06
 static int  parser_scion_looks_escaped( parser *p)
 {
    unsigned char   c;
27dc2200
 
fc1b9d06
    if( p->curr >= p->sentinel)
       return( 0);
    c = *p->curr;
27dc2200
 
fc1b9d06
    //
    // if {{ is followed immediately by something that looks escapish
    // like a backtick, then we assume it is escaped
    // (for documenting MulleScion itself ?)
    //
    return( c == '`' || c == '\\');
 }
776920e7
 
2995f4ad
 //
 // this will stop at '{{' or '{%' even if they are in the middle of
776920e7
 // quotes or comments. To print {{ use {{ "{{" }}
2995f4ad
 //
fc1b9d06
 static macro_type   parser_grab_text_until_scion_start( parser *p)
2995f4ad
 {
    unsigned char   c, d;
801f8317
    int             inquote;
fb5e6269
    macro_type      type;
27dc2200
 
776920e7
    assert( p->skipComments <= 0);
 
2995f4ad
    parser_memorize( p, &p->memo);
27dc2200
 
fb5e6269
    type    = garbage;
801f8317
    inquote = 0;
fb5e6269
    c       = p->curr > p->buf ? p->curr[ -1] : 0;
27dc2200
 
2995f4ad
    while( p->curr < p->sentinel)
    {
       d = c;
       c = *p->curr++;
801f8317
 
fb5e6269
       //
       // try to also find stray closers and error then, because it's easier
       // for template editing
       //
       switch( c)
       {
       case '\n' :  type = garbage; parser_nl( p); continue;
       case '"'  :  type = garbage; inquote = ! inquote; continue;
       default   :  type = garbage; continue;
       case '%'  :  type = command; break;
       case '#'  :  type = comment; break;
ef82a9e8
       case '{'  :  type = expression; break;
fc1b9d06
       case '}'  :  if( type == garbage)
                      continue;
                    if( parser_scion_looks_escaped( p))
                    {
                       type    = garbage;
                       continue;
                    }
da962797
                    // because of css with e.g. width=100%}, we let it slide
                    if( type == command)
                    {
                      parser_warning( p, "unexpected '%c}' without opener", d);
                      type    = garbage;
                      continue;
                    }
fc1b9d06
                    parser_error( p, "unexpected '%c}' without opener", d);
fb5e6269
       }
27dc2200
 
2995f4ad
       if( d == '{')
       {
fc1b9d06
          if( parser_scion_looks_escaped( p))
          {
             type    = garbage;
             continue;
          }
27dc2200
 
801f8317
          parser_memorize( p, &p->memo_scion);
fb5e6269
          p->curr -= 2;
          return( type);
2995f4ad
       }
    }
27dc2200
 
2995f4ad
    return( eof);
 }
 
 
04373918
 //
 // assume whitespace and leading comments have been removed
 //
 static macro_type   parser_next_scion_end( parser *p)
2995f4ad
 {
    unsigned char   c, d;
27dc2200
 
04373918
    if( p->curr + 2 > p->sentinel)
       return( eof);
27dc2200
 
04373918
    d = *p->curr++;
    c = *p->curr++;
27dc2200
 
04373918
    if( c == '}')
       switch( d)
fb5e6269
       {
04373918
       case '}' : return( expression);
       case '%' : return( command);
       case '#' : return( comment);
fb5e6269
       }
04373918
    p->curr -= 2;
    return( garbage);
2995f4ad
 }
 
04373918
 
776920e7
 # pragma mark -
 # pragma mark whitespace
 
 static void   parser_skip_whitespace( parser *p)
 {
    unsigned char   c;
27dc2200
 
776920e7
    for( ; p->curr < p->sentinel; p->curr++)
    {
       c = *p->curr;
       switch( c)
       {
       case '\n' :
          parser_nl( p);
          break;
27dc2200
 
776920e7
       case '#'  :
          if( p->skipComments > 0)
          {
             parser_skip_after_newline( p);
             --p->curr; // because we add it again in for
             break;
          }
          return;
27dc2200
 
776920e7
       default :
          if( c > ' ')
             return;
       }
    }
 }
 
2995f4ad
 
04373918
 static void   parser_skip_whitespace_and_comments_always( parser *p)
 {
    unsigned char   c;
27dc2200
 
04373918
    for( ; p->curr < p->sentinel; p->curr++)
    {
       c = *p->curr;
       switch( c)
       {
       case '\n' :
          parser_nl( p);
          break;
27dc2200
 
04373918
       case '#'  :
          parser_skip_after_newline( p);
          --p->curr; // because we add it again in for
          break;
27dc2200
 
04373918
       default :
          if( c > ' ')
             return;
       }
    }
 }
 
 
2995f4ad
 static void   parser_skip_white_if_terminated_by_newline( parser *p)
 {
    parser_memo   memo;
    unsigned char   c;
27dc2200
 
776920e7
    assert( p->skipComments <= 0);
 
2995f4ad
    parser_memorize( p, &memo);
27dc2200
 
2995f4ad
    for( ; p->curr < p->sentinel;)
    {
       c = *p->curr++;
       if( c == '\n')
       {
801f8317
          parser_nl( p);
2995f4ad
          return;
       }
27dc2200
 
2995f4ad
       if( c > ' ')
       {
          parser_recall( p, &memo);
          break;
       }
    }
 }
 
 
2523ba40
 static void   parser_skip_after_newline( parser *p)
 {
    unsigned char   c;
27dc2200
 
2523ba40
    for( ; p->curr < p->sentinel;)
    {
       c = *p->curr++;
       if( c == '\n')
c533990f
       {
          parser_nl( p);
2523ba40
          break;
c533990f
       }
2523ba40
    }
 }
 
 
04373918
 static void   parser_skip_white_until_after_newline( parser *p)
2995f4ad
 {
    unsigned char   c;
27dc2200
 
2995f4ad
    for( ; p->curr < p->sentinel;)
    {
       c = *p->curr++;
       if( c == '\n')
       {
801f8317
          parser_nl( p);
776920e7
          break;
2995f4ad
       }
27dc2200
 
2995f4ad
       if( c > ' ')
       {
          p->curr--;
          break;
       }
    }
 }
 
 
04373918
 # pragma mark -
 # pragma mark scion tags
 
776920e7
 static macro_type   parser_skip_text_until_scion_end( parser *p, int type)
 {
    unsigned char   c;
    unsigned char   d;
    int             inquote;
27dc2200
 
776920e7
    assert( p->skipComments <= 0);
27dc2200
 
776920e7
    inquote = 0;
    c       = 0;
27dc2200
 
776920e7
    for( ; p->curr < p->sentinel;)
    {
       d = c;
       c = *p->curr++;
       if( c == '\n')
       {
          parser_nl( p);
          continue;
       }
       if( c == '"')
          inquote = ! inquote;
       if( inquote)
          continue;
27dc2200
 
776920e7
       if( c == '}')
fec98eee
       {
          if( d == type || type == 0xFF)
          {
             switch( d)
             {
                case '%' : return( command);
                case '#' : return( comment);
                case '}' : return( expression);
             }
          }
       }
776920e7
    }
    return( eof);
 }
 
 
2995f4ad
 // we consume white space to
 static void   parser_adjust_memo_to_end_of_previous_line( parser *p, parser_memo *memo)
 {
    unsigned char  *s;
    unsigned char  c;
27dc2200
 
2995f4ad
    s = memo->curr;
    while( s > p->buf)
    {
       c = *--s;
       if( c == '\n')
       {
          memo->curr = s + 1;
          break;
       }
27dc2200
 
2995f4ad
       if( c > ' ')  // any kind of non-whitespace preserves other whitespace
          break;
    }
 }
 
 
 // just a simple heuristic to grab enough characters
 static BOOL   parser_grab_text_until_number_end( parser *p)
 {
    unsigned char   c;
    BOOL            isFloat;
27dc2200
 
2995f4ad
    parser_memorize( p, &p->memo);
 
    isFloat = NO;
    for( ; p->curr < p->sentinel; p->curr++)
    {
       c = *p->curr;
       if( c >= '0' && c <= '9')
          continue;
 
       if( c == '+' || c == '-')
          continue;
 
       if( c == '.' || c == 'e')
       {
          isFloat = YES;
          continue;
       }
       break;
    }
    return( isFloat);
 }
 
 
801f8317
 static int   parser_grab_text_until_selector_end( parser *p, int partial)
2995f4ad
 {
    unsigned char   c;
27dc2200
 
2995f4ad
    parser_memorize( p, &p->memo);
27dc2200
 
2995f4ad
    for( ; p->curr < p->sentinel; p->curr++)
    {
       c = *p->curr;
27dc2200
 
2995f4ad
       if( c >= '0' && c <= '9')
       {
          if( p->memo.curr  == p->curr)
             break;
          continue;
       }
27dc2200
 
2995f4ad
       if( c >= 'A' && c <= 'Z')
          continue;
27dc2200
 
2995f4ad
       if( c >= 'a' && c <= 'z')
          continue;
27dc2200
 
2995f4ad
       if( c == '_')
          continue;
27dc2200
 
2995f4ad
       if( c == ':')
       {
801f8317
          if( ! partial)
             continue;
2995f4ad
          p->curr++;
          return( 1);
       }
27dc2200
 
2995f4ad
       break;
    }
    return( 0);
 }
 
 
 static void   parser_grab_text_until_key_path_end( parser *p)
 {
    unsigned char   c;
27dc2200
 
2995f4ad
    parser_memorize( p, &p->memo);
27dc2200
 
2995f4ad
    for( ; p->curr < p->sentinel; p->curr++)
    {
       c = *p->curr;
27dc2200
 
2995f4ad
       //
       // stuff that keypaths can't start with (but may contain)
       // MulleScion uses the '#'
       //
       if( (c >= '0' && c <= '9') ||  c == '#' || c == '@' || c == '.' || c == ':')
       {
          if( p->memo.curr == p->curr)
             break;
          continue;
       }
27dc2200
 
2995f4ad
       if( c >= 'A' && c <= 'Z')
          continue;
27dc2200
 
2995f4ad
       if( c >= 'a' && c <= 'z')
          continue;
27dc2200
 
2995f4ad
       if( c == '_')
          continue;
       break;
    }
 }
 
 
 static void   parser_grab_text_until_identifier_end( parser *p)
 {
    unsigned char   c;
27dc2200
 
2995f4ad
    parser_memorize( p, &p->memo);
27dc2200
 
2995f4ad
    for( ; p->curr < p->sentinel; p->curr++)
    {
       c = *p->curr;
27dc2200
 
2995f4ad
       if( c >= '0' && c <= '9')
       {
          if( p->memo.curr == p->curr)
             break;
          continue;
       }
27dc2200
 
2995f4ad
       if( c >= 'A' && c <= 'Z')
          continue;
27dc2200
 
2995f4ad
       if( c >= 'a' && c <= 'z')
          continue;
27dc2200
 
2995f4ad
       if( c == '_')
          continue;
27dc2200
 
2995f4ad
       break;
    }
 }
 
 
801f8317
 static int   parser_grab_text_until_command( parser *p, char *command)
 {
    unsigned char   c;
    unsigned char   d;
    int             stage;
    size_t          len;
    int             inquote;
    parser_memo     memo;
27dc2200
 
801f8317
    inquote = 0;
    len   = strlen( command);
    stage = -2;
    c     = 0;
    for( ; p->curr < p->sentinel;)
    {
       d = c;
       c = *p->curr++;
       if( c == '\n')
          parser_nl( p);
27dc2200
 
801f8317
       if( c == '"')
          inquote = ! inquote;
       if( inquote)
       {
          stage = -2;
          continue;
       }
27dc2200
 
801f8317
       switch( stage)
       {
       case -2 :
          if( c == '%')
             if( d == '{')
             {
                parser_memorize( p, &memo);
                stage++;
             }
          continue;
27dc2200
 
801f8317
       case -1 :
          if( c <= ' ')
             continue;
          ++stage;
27dc2200
 
801f8317
       default :
04373918
          if( (size_t) stage == len)
801f8317
          {
             if( c <= ' ' || c == '%')
             {
                parser_recall( p, &memo);
                p->curr -= 2;
                return( 1);
             }
          }
          else
             if( c == command[ stage])
             {
                ++stage;
                continue;
             }
          stage = -2;
          continue;
       }
    }
    return( 0);
 }
 
 
ef13e42e
 static int   parser_grab_text_until_quote( parser *p)
2995f4ad
 {
    unsigned char   c;
    int             escaped;
27dc2200
 
2995f4ad
    parser_memorize( p, &p->memo);
27dc2200
 
2995f4ad
    escaped = 0;
27dc2200
 
2995f4ad
    for( ; p->curr < p->sentinel;)
    {
       c = *p->curr++;
       if( c == '\n')
801f8317
          parser_nl( p);
27dc2200
 
2995f4ad
       if( escaped)
       {
          escaped = 0;
          continue;
       }
       if( c == '\\')
       {
          escaped  = 1;
          continue;
       }
       if( c == '"')
       {
          p->curr--;
ef13e42e
          return( 1);
2995f4ad
       }
    }
ef13e42e
    return( 0);
2995f4ad
 }
 
 
 static NSString   * NS_RETURNS_RETAINED parser_get_memorized_retained_string( parser_memo *start, parser_memo *end)
 {
    NSInteger    length;
    NSString     *s;
    NSData       *data;
27dc2200
 
2995f4ad
    length = end->curr - start->curr ;
    if( length <= 0)
       return( nil);
27dc2200
 
2995f4ad
    data = [[NSData alloc] initWithBytesNoCopy:start->curr
ef903235
                                        length:(NSUInteger) length
2995f4ad
                                  freeWhenDone:NO];
    s = [[NSString alloc] initWithData:data
                              encoding:NSUTF8StringEncoding];
    [data release];
27dc2200
 
2995f4ad
    return( s);
 }
 
 
27dc2200
 static NSString   * NS_RETURNS_RETAINED parser_get_retained_string( parser *p)
2995f4ad
 {
c533990f
    NSUInteger      length;
    NSUInteger      unescaped_length;
    NSString        *s;
    unsigned char   *unescaped;
27dc2200
 
2995f4ad
    length = p->curr - p->memo.curr ;
    if( ! length)
       return( nil);
27dc2200
 
c533990f
    unescaped = unescaped_string_if_needed( p->memo.curr, length, &unescaped_length);
27dc2200
 
c533990f
    if( unescaped)
    {
       s = [[NSString alloc] initWithBytesNoCopy:unescaped
                                          length:unescaped_length
                                        encoding:NSUTF8StringEncoding
                                    freeWhenDone:YES];
       return( s);
    }
27dc2200
 
 
2995f4ad
    s = [[NSString alloc] initWithBytes:p->memo.curr
                                 length:length
                               encoding:NSUTF8StringEncoding];
    return( s);
 }
 
 
 static NSString   *parser_get_string( parser *p)
 {
    return( [parser_get_retained_string( p) autorelease]);
 }
 
 
 static unsigned char   parser_peek_character( parser *p)
 {
    return( p->curr < p->sentinel ? *p->curr : 0);
 }
 
 
c1dca06f
 static unsigned char   parser_peek2_character( parser *p)
 {
    return( p->curr + 1 < p->sentinel ? p->curr[ 1] : 0);
 }
 
 
c533990f
 static inline unsigned char   parser_next_character( parser *p)
 {
    unsigned char   c;
27dc2200
 
c533990f
    if( p->curr >= p->sentinel)
       return( 0);
fec98eee
 
c533990f
    c = *p->curr++;
    if( c == '\n')
       parser_nl( p);
    return( c);
 }
 
 
 // unused it seems
2995f4ad
 static void   parser_undo_character( parser *p)
 {
    if( p->curr <= p->buf)
       parser_error( p, "internal buffer underflow");
c533990f
    if( *--p->curr == '\n')
       p->lineNumber--;
2995f4ad
 }
 
 
c533990f
 static inline void  parser_skip_peeked_character( parser *p, char c)
2995f4ad
 {
c533990f
    assert( p->curr < p->sentinel);
    assert( *p->curr == c);
    p->curr++;
 }
 
 
 static inline void   parser_peek_expected_character( parser *p, char expect, char *error)
 {
    unsigned char   c;
27dc2200
 
c533990f
    if( p->curr >= p->sentinel)
       parser_error( p, "end of file reached, %s", error);
27dc2200
 
c533990f
    c = *p->curr;
    if( c != expect)
       parser_error( p, error);
    if( c == '\n')
       parser_nl( p);
 }
 
 
 static inline void   parser_next_expected_character( parser *p, char expect, char *error)
 {
    parser_peek_expected_character( p, expect, error);
    p->curr++;
2995f4ad
 }
 
 
99fb8c65
 # pragma mark -
 # pragma mark Simple Expressions
 
2995f4ad
 static NSString  *parser_do_key_path( parser *p)
 {
    NSString   *s;
27dc2200
 
2995f4ad
    parser_grab_text_until_key_path_end( p);
    s = parser_get_string( p);
    if( ! s)
c533990f
       parser_error( p, "a key path was expected");
2995f4ad
    parser_skip_whitespace( p);
    return( s);
 }
 
 
 static NSString  *parser_do_identifier( parser *p)
 {
    NSString   *s;
27dc2200
 
2995f4ad
    parser_grab_text_until_identifier_end( p);
    s = parser_get_string( p);
    if( ! s)
c533990f
       parser_error( p, "an identifier was expected");
2995f4ad
    parser_skip_whitespace( p);
    return( s);
 }
 
 
 static NSNumber *parser_do_number( parser *p)
 {
    NSString   *s;
    BOOL       isFloat;
27dc2200
 
2995f4ad
    isFloat = parser_grab_text_until_number_end( p);
    s       = parser_get_string( p);
    if( ! s)
c533990f
       parser_error( p, "a number was expected");
2995f4ad
    parser_skip_whitespace( p);
    if( isFloat)
       return( [NSNumber numberWithDouble:[s doubleValue]]);
    return( [NSNumber numberWithLongLong:[s longLongValue]]);
 }
 
 
 static NSString  *parser_do_string( parser *p)
 {
    NSString   *s;
27dc2200
 
2995f4ad
    NSCParameterAssert( parser_peek_character( p) == '"');
27dc2200
 
c533990f
    parser_skip_peeked_character( p, '\"');   // skip '"'
ef13e42e
    if( ! parser_grab_text_until_quote( p))
       parser_error( p, "a closing '\"' was expected");
 
2995f4ad
    s = parser_get_string( p);
c533990f
    parser_skip_peeked_character( p, '\"');   // skip '"'
2995f4ad
    parser_skip_whitespace( p);
27dc2200
 
2995f4ad
    return( s ? s : @"");
 }
 
99fb8c65
 # pragma mark -
 # pragma mark Expressions
2995f4ad
 
 static MulleScionExpression * NS_RETURNS_RETAINED  parser_do_expression( parser *p);
801f8317
 
08b5d36d
 static inline MulleScionExpression * NS_RETURNS_RETAINED  parser_do_unary_expression( parser *p);
2995f4ad
 
c533990f
 static NSMutableDictionary  *parser_do_dictionary( parser *p)
2995f4ad
 {
7e71be68
    NSMutableDictionary    *dict;
2995f4ad
    MulleScionExpression   *expr;
c533990f
    MulleScionExpression   *keyExpr;
2995f4ad
    unsigned char           c;
27dc2200
 
c533990f
    NSCParameterAssert( parser_peek_character( p) == '{');
    parser_skip_peeked_character( p, '{');   // skip '"'
27dc2200
 
c533990f
    dict = [NSMutableDictionary dictionary];
2995f4ad
    expr  = nil;
    for(;;)
    {
       parser_skip_whitespace( p);
       c = parser_peek_character( p);
c533990f
       if( c == '}')
2995f4ad
       {
c533990f
          parser_skip_peeked_character( p, '}');
2995f4ad
          break;
       }
27dc2200
 
2995f4ad
       if( c == ',')
       {
          if( ! expr)
c533990f
             parser_error( p, "a lonely comma in an array was found");
          parser_skip_peeked_character( p, ',');
2995f4ad
       }
       else
       {
          if( expr)
c533990f
             parser_error( p, "a comma or closing curly brackets was expected");
2995f4ad
       }
27dc2200
 
2995f4ad
       expr = parser_do_expression( p);
27dc2200
 
c533990f
       parser_skip_whitespace( p);
       c = parser_peek_character( p);
       if( c != ',')
          parser_error( p, "a comma and a key was expected");
 
       parser_skip_peeked_character( p, ',');
       keyExpr = parser_do_expression( p);
ef13e42e
       if( ! [keyExpr isDictionaryKey])
          parser_error( p, "a number or string as a key was expected");
c533990f
       [dict setObject:expr
ef13e42e
                forKey:[keyExpr value]];
       [keyExpr release];
c533990f
       [expr release];
    }
    return( dict);
 }
 
27dc2200
 
c533990f
 static NSMutableArray   *parser_do_array_or_arguments( parser *p, int allow_arguments)
 {
7e71be68
    NSMutableArray                   *array;
c533990f
    MulleScionExpression             *expr;
    MulleScionIdentifierExpression   *key;
    unsigned char                    c;
27dc2200
 
c533990f
    parser_skip_peeked_character( p, '(');
 
    array = [NSMutableArray array];
    expr  = nil;
    for(;;)
    {
       parser_skip_whitespace( p);
       c = parser_peek_character( p);
       if( c == ')')
       {
          parser_skip_peeked_character( p, ')');
          break;
       }
27dc2200
 
c533990f
       if( c == '=' && allow_arguments)
       {
          MulleScionExpression   *value;
27dc2200
 
c533990f
          if( ! expr)
             parser_error( p, "a lonely '=' without a key in an argument list was found");
          if( ! [expr isIdentifier])
             parser_error( p, "an identifier before '=' was expected");
 
          key  = (MulleScionIdentifierExpression *) [expr retain];
          [array removeLastObject];
 
          parser_skip_peeked_character( p, '=');
          value = parser_do_expression( p);
          if( ! value)
             parser_error( p, "a value after '=' was expected");
27dc2200
 
          expr = [MulleScionParameterAssignment newWithIdentifier:[key identifier]
                                               retainedExpression:value
                                                       lineNumber:[key lineNumber]];
c533990f
          [key release];
       }
       else
       {
          if( c == ',')
          {
             if( ! expr)
                parser_error( p, "a lonely comma in an array was found");
             parser_skip_peeked_character( p, ',');
          }
          else
          {
             if( expr)
                parser_error( p, "a comma or closing parenthesis was expected");
          }
27dc2200
 
c533990f
          expr = parser_do_expression( p);
       }
27dc2200
       
07d4a6c3
       NSCParameterAssert( [expr isKindOfClass:[MulleScionExpression class]]);
 
2995f4ad
       [array addObject:expr];
       [expr release];
    }
    return( array);
 }
 
 
c533990f
 static NSMutableArray   *parser_do_array( parser *p)
 {
    return( parser_do_array_or_arguments( p, NO));
 }
 
27dc2200
 
c533990f
 static NSMutableArray   *parser_do_arguments( parser *p)
 {
    return( parser_do_array_or_arguments( p, YES));
 }
 
27dc2200
 
2995f4ad
 static MulleScionMethod  * NS_RETURNS_RETAINED parser_do_method( parser *p)
 {
    NSMutableString        *selBuf;
bdb5913e
    NSString               *selName;
7e71be68
    NSMutableArray         *arguments;
2995f4ad
    NSUInteger             line;
7e71be68
    MulleScionExpression   *target;
    MulleScionExpression   *expr;
2995f4ad
    int                    hasColon;
    unsigned char          c;
27dc2200
 
c533990f
    parser_skip_peeked_character( p, '[');
2995f4ad
    parser_skip_whitespace( p);
27dc2200
 
7e71be68
    line   = p->memo.lineNumber;
    target = parser_do_expression( p);
27dc2200
 
801f8317
    hasColon = parser_grab_text_until_selector_end( p, YES);
2995f4ad
    selName  = parser_get_string( p);
    if( ! selName)
c533990f
       parser_error( p, "a selector was expected");
27dc2200
 
2995f4ad
    arguments = nil;
    if( hasColon)
    {
       arguments = [NSMutableArray array];
       selBuf    = [NSMutableString string];
       for( ;;)
       {
          [selBuf appendString:selName];
27dc2200
 
2995f4ad
          for(;;)
          {
             expr = parser_do_expression( p);
             [arguments addObject:expr];
             [expr release];
27dc2200
 
2995f4ad
             parser_skip_whitespace( p);
             c = parser_peek_character( p);
             if( c != ',')
                break;
 
             parser_error( p, "sorry but varargs isn't in the cards yet");
27dc2200
 
             // parser_skip_peeked_character( p, ',');
             // parser_skip_whitespace( p);
2995f4ad
          }
27dc2200
 
801f8317
          hasColon = parser_grab_text_until_selector_end( p, YES);
2995f4ad
          if( hasColon)
          {
             selName = parser_get_string( p);
             continue;
          }
          break;
       }
27dc2200
 
2995f4ad
       selName = selBuf;
    }
27dc2200
 
2995f4ad
    parser_skip_whitespace( p);
c533990f
    parser_next_expected_character( p, ']', "a closing ']' was expected");
27dc2200
 
2995f4ad
    return( [MulleScionMethod newWithRetainedTarget:target
                                         methodName:selName
                                          arguments:arguments
                                         lineNumber:line]);
 }
 
 
fec98eee
 static MulleScionObject  * NS_RETURNS_RETAINED   parser_expand_macro_with_arguments( parser *p,
08b5d36d
                                                               MulleScionMacro *macro,
                                                               NSArray *arguments,
                                                               NSUInteger line)
 {
7e71be68
    MulleScionTemplate  *body;
    NSDictionary        *parameters;
    MulleScionObject    *obj;
    NSAutoreleasePool   *pool;
08b5d36d
 
    pool       = [NSAutoreleasePool new];
801f8317
    parameters = [macro parametersWithArguments:arguments
                                       fileName:p->fileName
                                     lineNumber:p->memo.lineNumber];
    body       = [macro expandedBodyWithParameters:parameters
                                          fileName:p->fileName
                                        lineNumber:p->memo.lineNumber];
27dc2200
 
08b5d36d
    // so hat do we do with the body now ?
    // snip off the head
b8c659f4
    obj = [body behead];
08b5d36d
    // the tricky thing is, that 'obj' is now not autoreleased anymore
    // while body is
    [pool release];
 
    return( obj);
 }
 
 
fec98eee
 static MulleScionFunction  * NS_RETURNS_RETAINED _parser_do_function( parser *p, NSString *identifier, NSArray *arguments)
 {
    MulleScionMacro   *macro;
27dc2200
 
fec98eee
    macro = [p->tables.macroTable objectForKey:identifier];
    if( macro)
       parser_error( p, "referencing a macro in an expression is not allowed");
27dc2200
 
b8c659f4
    if( p->environment & MULLESCION_DUMP_MACROS)
fec98eee
       fprintf( stderr, "referencing function \"%s\"\n", [identifier cString]);
27dc2200
 
fec98eee
    return( [MulleScionFunction newWithIdentifier:identifier
                                        arguments:arguments
                                       lineNumber:p->memo.lineNumber]);
 }
 
 
08b5d36d
 static MulleScionObject  * NS_RETURNS_RETAINED parser_do_function_or_macro( parser *p, NSString *identifier)
2995f4ad
 {
fec98eee
    NSArray           *arguments;
    MulleScionMacro   *macro;
27dc2200
 
2995f4ad
    NSCParameterAssert( parser_peek_character( p) == '(');
27dc2200
 
c533990f
    arguments       = parser_do_arguments( p);
fec98eee
    macro           = [p->tables.macroTable objectForKey:identifier];
08b5d36d
    p->wasMacroCall = macro != nil;
fec98eee
    if( p->wasMacroCall)
08b5d36d
       return( parser_expand_macro_with_arguments( p, macro, arguments, p->memo.lineNumber));
 
fec98eee
    return( _parser_do_function( p, identifier, arguments));
08b5d36d
 }
 
 
 static MulleScionFunction  * NS_RETURNS_RETAINED parser_do_function( parser *p, NSString *identifier)
 {
fec98eee
    NSArray   *arguments;
27dc2200
 
fec98eee
    arguments = parser_do_arguments( p);
    return( _parser_do_function( p, identifier, arguments));
 }
 
 
 
 static MulleScionFunction  * NS_RETURNS_RETAINED __parser_do_macro( parser *p, NSString *identifier)
 {
    NSArray   *arguments;
27dc2200
 
c533990f
    arguments = parser_do_arguments( p);
2995f4ad
    return( [MulleScionFunction newWithIdentifier:identifier
                                        arguments:arguments
                                       lineNumber:p->memo.lineNumber]);
 }
 
 
c533990f
 //static MulleScionParameterAssignment  * NS_RETURNS_RETAINED parser_do_assignment( parser *p, NSString *identifier)
 //{
 //   MulleScionExpression  *expr;
27dc2200
 //
c533990f
 //   NSCParameterAssert( parser_peek_character( p) == '=');
 //   parser_next_character( p);
 //   parser_skip_whitespace( p);
27dc2200
 //
c533990f
 //   expr = parser_do_expression( p);
 //   return( [MulleScionParameterAssignment newWithIdentifier:identifier
 //                                       retainedExpression:expr
 //                                               lineNumber:p->memo.lineNumber]);
 //}
2995f4ad
 
801f8317
 
 static MulleScionIndexing  * NS_RETURNS_RETAINED parser_do_indexing( parser *p,
                                                                      MulleScionExpression * NS_CONSUMED left,
                                                                      MulleScionExpression * NS_CONSUMED right)
 {
    parser_skip_whitespace( p);
c533990f
    parser_next_expected_character( p, ']', "a closing ']' was expected");
27dc2200
 
801f8317
    return( [MulleScionIndexing newWithRetainedLeftExpression:left
                                      retainedRightExpression:right
                                                   lineNumber:p->memo.lineNumber]);
 }
 
fec98eee
 
801f8317
 static MulleScionConditional  * NS_RETURNS_RETAINED parser_do_conditional( parser *p,
                                                                            MulleScionExpression * NS_CONSUMED left,
                                                                            MulleScionExpression * NS_CONSUMED middle)
 {
    MulleScionExpression   *right;
27dc2200
 
801f8317
    parser_skip_whitespace( p);
c533990f
    parser_next_expected_character( p, ':', "a conditional ':' was expected");
 
801f8317
    right = parser_do_expression( p);
    return( [MulleScionConditional newWithRetainedLeftExpression:left
                                        retainedMiddleExpression:middle
                                         retainedRightExpression:right
                                                      lineNumber:p->memo.lineNumber]);
 }
 
 
776920e7
 static MulleScionLog  * NS_RETURNS_RETAINED parser_do_log( parser *p, NSUInteger line)
 {
    MulleScionExpression  *expr;
27dc2200
 
776920e7
    parser_skip_whitespace( p);
27dc2200
 
776920e7
    expr = parser_do_unary_expression( p);
    return( [MulleScionLog newWithRetainedExpression:expr
                                          lineNumber:line]);
 }
 
 
801f8317
 static MulleScionNot  * NS_RETURNS_RETAINED parser_do_not( parser *p)
 {
    MulleScionExpression  *expr;
27dc2200
 
801f8317
    parser_skip_whitespace( p);
 
    // gives not implicit precedence over and / or
    expr = parser_do_unary_expression( p);
    return( [MulleScionNot newWithRetainedExpression:expr
                                          lineNumber:p->memo.lineNumber]);
 }
 
 
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_parenthesized_expression(  parser *p)
 {
    MulleScionExpression  *expr;
27dc2200
 
c533990f
    parser_skip_peeked_character( p, '(');
801f8317
 
    expr = parser_do_expression( p);
27dc2200
 
801f8317
    parser_skip_whitespace( p);
c533990f
    parser_next_expected_character( p, ')', "a closing ')' was expected. (Hint: prefix arrays with @)");
27dc2200
 
801f8317
    return( expr);
 }
 
 
 static MulleScionSelector * NS_RETURNS_RETAINED  parser_do_selector(  parser *p)
 {
c533990f
    NSString   *selectorName;
27dc2200
 
801f8317
    parser_skip_whitespace( p);
 
c533990f
    parser_next_expected_character( p, '(', "a '(' after the selector keyword was expected");
801f8317
    parser_skip_whitespace( p);
 
    parser_grab_text_until_selector_end( p, NO);
    selectorName  = parser_get_string( p);
    if( ! [selectorName length])
c533990f
       parser_error(p, "a selector name was expected");
27dc2200
 
801f8317
    parser_skip_whitespace( p);
c533990f
    parser_next_expected_character( p, ')', "a closing ')' after the selector name was expected");
801f8317
 
    return( [MulleScionSelector newWithString:selectorName
                                   lineNumber:p->memo.lineNumber]);
 }
 
 
c533990f
 //case '=' : parser_next_character( p);
 //c = parser_peek_character( p);
 //parser_undo_character( p);
 //if( c == '=')
 //break;
 //return( parser_do_assignment( p, s));
08b5d36d
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_unary_expression_or_macro( parser *p, int allowMacroCall)
2995f4ad
 {
7e71be68
    NSString              *s;
    unsigned char         c;
    MulleScionExpression  *expr;
    int                   hasAt;
27dc2200
 
2995f4ad
    parser_skip_whitespace( p);
aba423bf
 
    c     = parser_peek_character( p);
801f8317
    hasAt = (c == '@');
    if( hasAt)  // skip adorning '@'s
2995f4ad
    {
c533990f
       parser_skip_peeked_character( p, '@');
2995f4ad
       c = parser_peek_character( p);
    }
27dc2200
 
2995f4ad
    switch( c)
    {
c533990f
    case '!' : parser_skip_peeked_character( p, '!');
801f8317
               return( parser_do_not( p));
    case '"' : return( [MulleScionString newWithString:parser_do_string( p)
                                            lineNumber:p->memo.lineNumber]);
    case '0' : case '1' : case '2' : case '3' : case '4' :
    case '5' : case '6' : case '7' : case '8' : case '9' :
    case '+' : case '-' : case '.' :  // valid starts of FP nrs
       return( [MulleScionNumber newWithNumber:parser_do_number( p)
                                    lineNumber:p->memo.lineNumber]);
c533990f
    case '{' :  if( hasAt)
                   return( [MulleScionDictionary newWithDictionary:parser_do_dictionary( p)
                                                        lineNumber:p->memo.lineNumber]);
                break;
801f8317
    case '(' :  if( hasAt)
                   return( [MulleScionArray newWithArray:parser_do_array( p)
                                              lineNumber:p->memo.lineNumber]);
                 return( parser_do_parenthesized_expression( p));
    case '[' : return( parser_do_method( p));
2995f4ad
    }
27dc2200
 
801f8317
    // this laymes
2995f4ad
    s = parser_do_key_path( p);
aba423bf
    while( [s hasPrefix:@"self."])
       s = [s substringFromIndex:5];
27dc2200
 
2995f4ad
    if( [s isEqualToString:@"nil"])
       return( [MulleScionNumber newWithNumber:nil
                                    lineNumber:p->memo.lineNumber]);
    if( [s isEqualToString:@"YES"])
       return( [MulleScionNumber newWithNumber:[NSNumber numberWithBool:YES]
                                    lineNumber:p->memo.lineNumber]);
    if( [s isEqualToString:@"NO"])
       return( [MulleScionNumber newWithNumber:[NSNumber numberWithBool:NO]
                                    lineNumber:p->memo.lineNumber]);
801f8317
    if( [s isEqualToString:@"not"])
       return( parser_do_not( p));
27dc2200
 
801f8317
    if( [s isEqualToString:@"selector"])
       return( parser_do_selector( p));
27dc2200
 
2995f4ad
    parser_skip_whitespace( p);
    c = parser_peek_character( p);
    switch( c )
    {
08b5d36d
    case '(' : if( allowMacroCall)
                  return( parser_do_function_or_macro( p, s));
               return( parser_do_function( p, s));
    default  : break;
2995f4ad
    }
08b5d36d
 
fec98eee
    expr = [p->tables.definitionTable objectForKey:s];
08b5d36d
    if( expr)
       return( [expr copyWithZone:NULL]);
 
    return( [MulleScionVariable newWithIdentifier:s
                                       lineNumber:p->memo.lineNumber]);
2995f4ad
 }
 
 
08b5d36d
 static inline MulleScionExpression * NS_RETURNS_RETAINED  parser_do_unary_expression( parser *p)
 {
    return( (MulleScionExpression *) parser_do_unary_expression_or_macro( p, NO));
 }
 
 
801f8317
 static BOOL  parser_next_matching_string( parser *p, char *expect, size_t len_expect)
 {
    parser_memo   memo;
    size_t        len;
27dc2200
 
801f8317
    parser_memorize( p, &memo);
    parser_grab_text_until_identifier_end( p);
    len = p->curr - memo.curr;
    if( len != len_expect)
    {
       parser_recall( p, &memo);
       return( NO);
    }
 
    return( ! memcmp( (char *) memo.curr, expect, len_expect));
 }
 
 
 static MulleScionComparisonOperator  parser_check_comparison_op( parser *p, char c)
 {
    char   d;
    BOOL   flag;
27dc2200
 
801f8317
    d = parser_next_character( p);
    c = parser_peek_character( p);
    flag = (c == '=');
    if( flag)
c533990f
       parser_skip_peeked_character( p, '=');
27dc2200
 
801f8317
    switch( d)
    {
    case '<' : return( flag ? MulleScionLessThanOrEqualTo : MulleScionLessThan);
    case '>' : return( flag ? MulleScionGreaterThanOrEqualTo : MulleScionGreaterThan);
    case '!' : if( ! flag) break; return( MulleScionNotEqual);
    }
 
c533990f
    parser_error( p, "an unexpected character %c was found", d);
 }
 
 
 static MulleScionComparisonOperator  parser_check_equal_or_set_op( parser *p, char c)
 {
    c = parser_peek2_character( p);
    if( c != '=')
       return( MulleScionNoComparison);
27dc2200
 
c533990f
    parser_skip_peeked_character( p, '=');
    parser_skip_peeked_character( p, '=');
    return( MulleScionEqual);
801f8317
 }
 
 
08b5d36d
 static MulleScionExpression * NS_RETURNS_RETAINED  _parser_do_expression( parser *p, MulleScionExpression *left)
2995f4ad
 {
ef903235
    MulleScionExpression           *right;
27dc2200
    char                           operator;
    MulleScionComparisonOperator   comparator;
512499f0
    MulleScionDot                  *dot;
3de6fc67
 
07d4a6c3
    NSCParameterAssert( [left isKindOfClass:[MulleScionExpression class]]);
    
801f8317
    parser_skip_whitespace( p);
27dc2200
 
2995f4ad
    /* get the operator */
    operator = parser_peek_character( p);
    switch( operator)
    {
801f8317
    default  : return( left);
    case '&' : if( ! parser_next_matching_string( p, "&&", 2))
                   return( left);
                operator = 'a';
                break;
    case '|' : if( parser_next_matching_string( p, "||", 2))
                  operator = 'o';
               else
c533990f
                  parser_skip_peeked_character( p, '|');
801f8317
               break;
    case 'a' : if( ! parser_next_matching_string( p, "and", 3))
                  return( left);
               break;
    case 'o' : if( ! parser_next_matching_string( p, "or", 2))
                  return( left);
               break;
    case '<' :
    case '>' :
27dc2200
    case '!' : operator = (char) parser_check_comparison_op( p, operator);
c533990f
               break;
27dc2200
 
    case '=' : comparator = parser_check_equal_or_set_op( p, operator);
               if( comparator == MulleScionNoComparison)
c533990f
                  return( left);
27dc2200
               operator = (char) comparator;
c533990f
               break;
801f8317
    case '?' :
    case '.' : // the irony, that I have to support "modern" ObjC to do C :)
c533990f
               parser_skip_peeked_character( p, operator);
               break;
          // this is problematic, because it could also be the start of a
          // unrelated method call... so lets request a new line between
    case '[' :
               if( p->lineNumber != [left lineNumber])
                  return( left);
               parser_skip_peeked_character( p, operator);
               break;
2995f4ad
    }
27dc2200
 
2995f4ad
    parser_skip_whitespace( p);
27dc2200
 
2995f4ad
    right = parser_do_expression( p);
27dc2200
 
2995f4ad
    switch( operator)
    {
801f8317
    default :
       return( [MulleScionComparison newWithRetainedLeftExpression:left
                                           retainedRightExpression:right
                                                        comparison:operator
                                                        lineNumber:p->memo.lineNumber]);
    case 'a' :
       return( [MulleScionAnd newWithRetainedLeftExpression:left
                                    retainedRightExpression:right
                                                 lineNumber:p->memo.lineNumber]);
    case 'o' :
       return( [MulleScionOr newWithRetainedLeftExpression:left
                                   retainedRightExpression:right
                                                lineNumber:p->memo.lineNumber]);
    case '[' :
3de6fc67
          left = parser_do_indexing( p, left, right);
          return( _parser_do_expression( p, left));
801f8317
 
    case '?' :
       return( parser_do_conditional( p, left, right));
27dc2200
 
801f8317
    case '|' :
       if( ! [right isMethod] && ! [right isPipe] && ! [right isIdentifier])
c533990f
          parser_error( p, "an identifier was expected after '|'");
801f8317
       return( [MulleScionPipe newWithRetainedLeftExpression:left
                                     retainedRightExpression:right
                                                  lineNumber:p->memo.lineNumber]);
    case '.' :
       if( ! [right isMethod] && ! [right isPipe] && ! [right isDot] && ! [right isIdentifier])
c533990f
          parser_error( p, "an identifier was expected after '.'");
512499f0
 
       dot = [MulleScionDot newWithRetainedLeftExpression:left
                                  retainedRightExpression:right
                                               lineNumber:p->memo.lineNumber];
 
       // limited precedence code for '|'
       if( ! [right isPipe])
          return( dot);
       return( [(MulleScionPipe *) right hierarchicalExchange:dot]);
2995f4ad
    }
    return( nil);  // can't happen
 }
 
08b5d36d
 // this is the path to arithmetic expression also, but DO NOT WANT right now
c533990f
 //
08b5d36d
 static MulleScionExpression * NS_RETURNS_RETAINED  parser_do_expression( parser *p)
 {
    MulleScionExpression  *left;
07d4a6c3
    MulleScionExpression  *expr;
27dc2200
 
08b5d36d
    left = parser_do_unary_expression( p);
07d4a6c3
    expr = _parser_do_expression( p, left);
    NSCParameterAssert( [expr isKindOfClass:[MulleScionExpression class]]);
 
    return( expr);
08b5d36d
 }
 
 
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_expression_or_macro( parser *p)
 {
    MulleScionObject  *left;
27dc2200
 
08b5d36d
    left = parser_do_unary_expression_or_macro( p, YES);
    if( p->wasMacroCall)
       return( left);
27dc2200
 
08b5d36d
    return( _parser_do_expression( p, (MulleScionExpression *) left));
 }
 
2995f4ad
 
801f8317
 static MulleScionObject  *_parser_next_object( parser *p,  MulleScionObject *owner, macro_type *last_type);
 
 static inline MulleScionObject  *parser_next_object( parser *p,  MulleScionObject *owner, macro_type *last_type)
 {
    NSAutoreleasePool   *pool;
    MulleScionObject    *curr;
27dc2200
 
801f8317
    pool  = [NSAutoreleasePool new];
    curr  = _parser_next_object( p, owner, last_type);
    // we know that curr has been retained by owner, so don't need to "save"
    [pool release];
27dc2200
 
801f8317
    return( curr);
 }
 
776920e7
 
801f8317
 static MulleScionObject  *_parser_next_object_after_extend( parser *p, macro_type *last_type);
 
 static inline MulleScionObject  *parser_next_object_after_extend( parser *p, macro_type *last_type)
 {
    NSAutoreleasePool   *pool;
    MulleScionObject    *curr;
27dc2200
 
801f8317
    pool  = [NSAutoreleasePool new];
    curr  = _parser_next_object_after_extend( p, last_type);
    [curr retain];
    [pool release];
27dc2200
 
801f8317
    return( [curr autorelease]);
 }
2995f4ad
 
 
04373918
 static char  *_macro_type_names[] =
 {
    "end of file",
    "}}",
    "%}",
    "#}",
    "garbage"
 };
 
 static char  **macro_type_names = &_macro_type_names[ 1];
 
 
 static void   parser_error_different_closer( parser *p, macro_type found, macro_type expected)
 {
    assert( found >= -1 && found <= 3);
    assert( expected >= -1 && expected <= 3);
27dc2200
 
04373918
    if( found == garbage)
       parser_error( p, "a '%s' was expected",
                macro_type_names[ expected]);
    else
       parser_error( p, "a '%s' was expected, but '%s' was found",
                macro_type_names[ expected],
                macro_type_names[ found]);
 }
 
 
 static inline void   parser_error_if_different_closer( parser *p, macro_type found, macro_type expected)
 {
    if( found != expected)
       parser_error_different_closer( p, found, expected);
 }
 
 
2995f4ad
 static void   parser_finish_comment( parser *p)
 {
    macro_type  end_type;
27dc2200
 
801f8317
    end_type = parser_skip_text_until_scion_end( p, '#');
2995f4ad
    parser_skip_white_if_terminated_by_newline( p);
04373918
    parser_error_if_different_closer( p, end_type, comment);
2995f4ad
 }
 
 
 static void   parser_finish_expression( parser *p)
 {
    macro_type  end_type;
27dc2200
 
04373918
    parser_skip_whitespace_and_comments_always( p);
    end_type = parser_next_scion_end( p);
    parser_error_if_different_closer( p, end_type, expression);
2995f4ad
 }
 
 
 static void   parser_finish_command( parser *p)
 {
    macro_type  end_type;
27dc2200
 
04373918
    parser_skip_whitespace_and_comments_always( p);
    end_type = parser_next_scion_end( p);
    parser_skip_white_until_after_newline( p);
 
    parser_error_if_different_closer( p, end_type, command);
2995f4ad
 }
 
 
801f8317
 typedef enum
 {
ef13e42e
    ImplicitSetOpcode = 0,  // still used ??
801f8317
 
    BlockOpcode,
    DefineOpcode,
    ElseOpcode,
d7a43901
    ElseforOpcode,
801f8317
    EndblockOpcode,
    EndfilterOpcode,
    EndforOpcode,
    EndifOpcode,
    EndmacroOpcode,
    EndverbatimOpcode,
    EndwhileOpcode,
    ExtendsOpcode,
    FilterOpcode,
    ForOpcode,
    IfOpcode,
    IncludesOpcode,
776920e7
    LogOpcode,
801f8317
    MacroOpcode,
fee8e7fd
    RequiresOpcode,
801f8317
    SetOpcode,
    VerbatimOpcode,
d7a43901
    WhileOpcode,
801f8317
 } MulleScionOpcode;
 
 
 static char   *mnemonics[] =
 {
    "set",
27dc2200
 
801f8317
    "block",
    "define",
    "else",
d7a43901
    "elsefor",
801f8317
    "endblock",
    "endfilter",
    "endfor",
    "endif",
    "endverbatim"
    "endwhile",
    "extends",
    "filter",
    "for",
    "if",
    "includes",
776920e7
    "log",
801f8317
    "macro",
fee8e7fd
    "requires",
801f8317
    "set",
    "verbatim",
c533990f
    "while"
801f8317
 };
 
 // LAYME!!
 static int   _parser_opcode_for_string( parser *p, NSString *s)
 {
    NSUInteger  len;
27dc2200
 
801f8317
    len = [s length];
    switch( len)
    {
    case 2 : if( [s isEqualToString:@"if"]) return( IfOpcode); break;
fec98eee
    case 3 : if( [s isEqualToString:@"for"]) return( ForOpcode);
             if( [s isEqualToString:@"set"]) return( SetOpcode);
776920e7
             if( [s isEqualToString:@"log"]) return( LogOpcode); break;
801f8317
    case 4 : if( [s isEqualToString:@"else"]) return( ElseOpcode); break;
    case 5 : if( [s isEqualToString:@"endif"]) return( EndifOpcode);
             if( [s isEqualToString:@"while"]) return( WhileOpcode);
fec98eee
             if( [s isEqualToString:@"block"]) return( BlockOpcode);
             if( [s isEqualToString:@"macro"]) return( MacroOpcode); break;
801f8317
    case 6 : if( [s isEqualToString:@"define"]) return( DefineOpcode);
             if( [s isEqualToString:@"endfor"]) return( EndforOpcode);
             if( [s isEqualToString:@"filter"]) return( FilterOpcode); break;
d7a43901
    case 7 : if( [s isEqualToString:@"extends"]) return( ExtendsOpcode);
             if( [s isEqualToString:@"elsefor"]) return( ElseforOpcode); break;
801f8317
    case 8 : if( [s isEqualToString:@"endblock"]) return( EndblockOpcode);
             if( [s isEqualToString:@"endwhile"]) return( EndwhileOpcode);
             if( [s isEqualToString:@"endmacro"]) return( EndmacroOpcode);
             if( [s isEqualToString:@"includes"]) return( IncludesOpcode);
fee8e7fd
             if( [s isEqualToString:@"requires"]) return( RequiresOpcode);
801f8317
             if( [s isEqualToString:@"verbatim"]) return( VerbatimOpcode); break;
    case 9 : if( [s isEqualToString:@"endfilter"]) return( EndfilterOpcode); break;
    case 11: if( [s isEqualToString:@"endverbatim"]) return( EndverbatimOpcode); break;
    }
    return( -1);
 }
 
 
 static MulleScionOpcode   parser_opcode_for_string( parser *p, NSString *s)
 {
    int   opcode;
27dc2200
 
801f8317
    opcode = _parser_opcode_for_string( p, s);
    if( opcode < 0)
       return( ImplicitSetOpcode);
    return( (MulleScionOpcode) opcode);
 }
 
 
 static char  *parser_best_match_for_string( parser *p, NSString *s)
 {
    NSUInteger   length;
    char         *c_s;
    int          i;
27dc2200
 
801f8317
    s      = [s lowercaseString];
    length = [s length];
    c_s    = (char *) [s cString];
    while( length)
    {
04373918
       for( i = 0; i < (int) (sizeof( mnemonics) / sizeof( char *)); i++)
801f8317
          if( ! strncmp( mnemonics[ i], c_s, length))
             return( mnemonics[ i]);
27dc2200
 
801f8317
       --length;
    }
    return( NULL);
 }
 
 
86f84e10
 // grabs a whole block and put it into the table
2995f4ad
 static void  parser_do_whole_block_to_block_table( parser *p)
 {
801f8317
    MulleScionBlock      *block;
    MulleScionObject     *next;
    MulleScionObject     *node;
    NSString             *identifier;
    NSUInteger           stack;
    macro_type           last_type;
27dc2200
 
2995f4ad
    parser_skip_whitespace( p);
    identifier = parser_do_identifier( p);
    if( ! [identifier length])
c533990f
       parser_error( p, "an identifier was expected");
2995f4ad
    parser_finish_command( p);
27dc2200
 
d7a43901
    block = [MulleScionBlock newWithIdentifier:identifier
                                      fileName:p->fileName
                                    lineNumber:p->memo.lineNumber];
801f8317
    assert( block);
27dc2200
 
2995f4ad
    stack     = 1;
    last_type = command;
    for( node = block; next = parser_next_object( p, node, &last_type); node = next)
    {
       if( [next isBlock])
       {
          ++stack;
          continue;
       }
       if( [next isEndBlock])
       {
          if( ! --stack)
             break;
          continue;
       }
    }
801f8317
 
    if( ! next)
c533990f
       parser_error( p, "an endblock was expected");
27dc2200
 
fec98eee
    [p->tables.blockTable setObject:block
                             forKey:identifier];
801f8317
    [block release];
2995f4ad
 }
 
 
 static MulleScionBlock * NS_RETURNS_RETAINED  parser_do_block( parser *p, NSUInteger line)
 {
    NSString   *identifier;
27dc2200
 
7a19085b
    //
    // this is actually not really necessary, but I can't decide if to expand
    // the identifer in the macro or not. Lets have a use case first
    //
    if( p->inMacro)
       parser_error( p, "no block definitions in a macro definition.");
27dc2200
 
2995f4ad
    identifier = parser_do_identifier( p);
    if( ! [identifier length])
c533990f
       parser_error( p, "an identifier was expected");
27dc2200
 
2995f4ad
    return( [MulleScionBlock newWithIdentifier:identifier
801f8317
                                      fileName:p->fileName
2995f4ad
                                    lineNumber:line]);
 }
 
 
801f8317
 static void  parser_add_dependency( parser *p, NSString *fileName, NSString *include)
 {
    NSMutableSet  *set;
27dc2200
 
fec98eee
    if( ! p->tables.dependencyTable)
801f8317
       return;
27dc2200
 
fec98eee
    set = [p->tables.dependencyTable objectForKey:fileName];
801f8317
    if( ! set)
    {
       set = [NSMutableSet new];
fec98eee
       [p->tables.dependencyTable setObject:set
801f8317
                              forKey:fileName];
       [set release];
    }
    [set addObject:include];
 }
 
 
2523ba40
 static NSString  * NS_RETURNS_RETAINED parser_remove_hashbang_from_string_if_desired( NSString NS_CONSUMED *s)
 {
    NSRange   range;
    NSString  *memo;
27dc2200
 
2523ba40
    if( ! [s hasPrefix:@"#!"])
       return( s);
27dc2200
 
2523ba40
    range = [s rangeOfString:@"\n"];
    if( ! range.length)
       return( @"");
27dc2200
 
2523ba40
    memo = s;
    s    = [[s substringFromIndex:range.location + 1] retain];
    [memo release];
    return( s);
 }
 
fee8e7fd
 
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_requires( parser *p, NSUInteger line)
 {
    NSString               *identifier;
27dc2200
 
fee8e7fd
    parser_skip_whitespace( p);
27dc2200
 
fee8e7fd
    identifier = parser_do_string( p);
    if( ! [identifier length])
       parser_error( p, "a bundle identifier was expected as a quoted string");
27dc2200
 
fee8e7fd
    return( [MulleScionRequires newWithIdentifier:identifier
                                       lineNumber:line]);
 }
 
2995f4ad
 /*
  * How Extends works.
2523ba40
  * when you read a file, the parser collects statement for the template.
2995f4ad
  * when it finds the keyword "extends" it stops collecting for the the template
2523ba40
  * and builds up the blockTable, all other stuff is discarded. This works
2995f4ad
  * recursively.
  */
fec98eee
 static MulleScionObject * NS_RETURNS_RETAINED  _parser_do_includes( parser *p, BOOL allowConverter)
2995f4ad
 {
801f8317
    MulleScionTemplate     *inferior;
    MulleScionTemplate     *marker;
    NSString               *fileName;
    BOOL                   verbatim;
fec98eee
    NSString               *converter;
    SEL                    sel;
801f8317
    NSString               *s;
f958902e
    char                   *env;
    
2ae60c68
    if( p->inMacro)
       parser_error( p, "no including or extending in macro");
27dc2200
 
801f8317
    verbatim = NO;
fec98eee
    sel      = 0;
27dc2200
 
fec98eee
 retry:
2995f4ad
    parser_skip_whitespace( p);
801f8317
    if( parser_peek_character( p) != '"')
    {
fec98eee
       if( ! allowConverter)
c533990f
          parser_error( p, "a filename was expected as a quoted string");
27dc2200
 
fec98eee
       converter = parser_do_identifier( p);
       if( [converter isEqualToString:@"verbatim"])
       {
          verbatim  = YES;
          converter = nil;
          goto retry;
       }
f958902e
    
       if( p->environment & MULLESCION_ALLOW_GETENV_INCLUDES)
       {
          env = getenv( [converter cString]);
          if( env)
          {
             fileName = [NSString stringWithCString:env];
             goto env_string;
          }
       }
          
fec98eee
       //
       // markdown -> markdownedData or some other variety
       // (rarely useful)
       sel = NSSelectorFromString( converter);
       if( ! [NSData instancesRespondToSelector:sel])
f958902e
          parser_error( p, "unknown converter method \"%s\" (hint: use quoted strings for filenames)", [converter cString]);
801f8317
    }
27dc2200
 
2995f4ad
    fileName = parser_do_string( p);
    if( ! [fileName length])
c533990f
       parser_error( p, "a filename was expected as a quoted string");
27dc2200
 
f958902e
 env_string:
dc48b1ee
    if( p->environment & MULLESCION_DUMP_FILE_INCLUDES)
       fprintf( stderr, "-> opening \"%s\"\n", [fileName UTF8String]);
    
801f8317
    if( verbatim)
    {
       s = [[NSString alloc] initWithContentsOfFile:fileName];
       if( ! s)
          parser_error( p, "could not load include file \"%@\"", fileName);
27dc2200
 
b8c659f4
       if( ! (p->environment & MULLESCION_VERBATIM_INCLUDE_HASHBANG) &&
           ! (p->environment & MULLESCION_NO_HASHBANG))
       {
2523ba40
          s = parser_remove_hashbang_from_string_if_desired( s);
b8c659f4
       }
801f8317
       return( [MulleScionPlainText newWithRetainedString:s
                                               lineNumber:p->memo.lineNumber]);
    }
 
    parser_finish_command( p);
 
2995f4ad
 NS_DURING
    inferior = [p->self templateWithContentsOfFile:fileName
fec98eee
                                            tables:&p->tables
                                         converter:sel];
2995f4ad
 NS_HANDLER
60f90a83
    parser_error( p, "\n%@", [localException reason]);
2995f4ad
 NS_ENDHANDLER
    if( ! inferior)
801f8317
       parser_error( p, "could not load include file \"%@\"", fileName);
27dc2200
 
801f8317
    parser_add_dependency( p, p->fileName, [inferior fileName]);
27dc2200
 
2995f4ad
    //
    // make a marker, that we are back. If we are extending all but
    // the blocks are discarded. That's IMO OK
    //
27dc2200
 
2995f4ad
    marker = [[MulleScionTemplate alloc] initWithFilename:[p->self fileName]];
    [[inferior tail] appendRetainedObject:marker];
27dc2200
 
2995f4ad
    return( [inferior retain]);
 }
 
 
776920e7
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_includes( parser *p, BOOL allowVerbatim)
 {
    MulleScionObject   *obj;
    int                memo;
27dc2200
 
776920e7
    memo            = p->skipComments;
    p->skipComments = 0;
    obj             = _parser_do_includes( p, allowVerbatim);
    p->skipComments = memo;
 
    return( obj);
 }
 
 
 static MulleScionTemplate * NS_RETURNS_RETAINED  _parser_do_extends( parser *p)
2995f4ad
 {
    MulleScionTemplate    *inferior;
    macro_type            last_type;
    MulleScionObject      *obj;
27dc2200
 
776920e7
    inferior  = (MulleScionTemplate *) _parser_do_includes( p, NO);
2995f4ad
    last_type = command;
27dc2200
 
2995f4ad
    for(;;)
    {
       obj = parser_next_object_after_extend( p, &last_type);
       if( obj)
          [[inferior tail] appendRetainedObject:obj];
       if( last_type == eof)
          break;
    }
    return( inferior);
 }
 
 
776920e7
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_extends( parser *p)
 {
    MulleScionObject   *obj;
    int                memo;
27dc2200
 
776920e7
    memo            = p->skipComments;
    p->skipComments = 0;
    obj             = _parser_do_extends( p);
    p->skipComments = memo;
 
    return( obj);
 }
 
 
c58aaf8d
 #if 0
2995f4ad
 static MulleScionFunctionCall  * NS_RETURNS_RETAINED  parser_do_function_call( parser *p, NSString *identifier)
 {
    MulleScionExpression  *expr;
27dc2200
 
2995f4ad
    expr = parser_do_function( p, identifier);
    return( [MulleScionFunctionCall newWithRetainedExpression:expr
                                                   lineNumber:p->memo.lineNumber]);
 }
c58aaf8d
 #endif
2995f4ad
 
 
 static MulleScionMethodCall  *NS_RETURNS_RETAINED parser_do_method_call( parser *p, NSUInteger line)
 {
    MulleScionExpression   *expr;
27dc2200
 
2995f4ad
    expr = parser_do_method( p);
    return( [MulleScionMethodCall newWithRetainedExpression:expr
                                                 lineNumber:line]);
 }
 
d7a43901
 
c533990f
 typedef struct
 {
    MulleScionExpression   *lexpr;
    MulleScionExpression   *expr;
    char                   *separator;
    int                    allow_expr_only;
 } assignment_or_expr_info;
 
2995f4ad
 
c533990f
 static void  init_assignment_or_expr_info( assignment_or_expr_info *p, char *separator, int allow_expr_only)
801f8317
 {
c533990f
    p->lexpr           = nil;
    p->expr            = nil;
    p->separator       = separator;
    p->allow_expr_only = allow_expr_only;
 }
 
 
 static int   _parser_do_assignment_or_expr( parser *p, assignment_or_expr_info *info,  NSUInteger line)
 {
    parser_memo     memo;
    unsigned char   *s;
27dc2200
 
c533990f
    NSCParameterAssert( info->expr && ! info->lexpr);
27dc2200
 
c533990f
    parser_memorize( p, &memo);
    for( s = (unsigned char *) info->separator; *s; s++)
       if( *s != parser_next_character( p))
       {
          if( info->allow_expr_only)
          {
             parser_recall( p, &memo);
             return( NO);
          }
          parser_error( p, "\'%s\' expected", info->separator);
       }
27dc2200
 
c533990f
    if( ! [info->expr isLexpr])
       parser_error( p, "left side not an assignable expression");
27dc2200
 
c533990f
    info->lexpr = info->expr;
801f8317
    parser_skip_whitespace( p);
c533990f
    info->expr = parser_do_expression( p);
27dc2200
 
c533990f
    return( YES);
 }
 
 
 static int   parser_do_assignment_or_expr( parser *p, assignment_or_expr_info *info, NSUInteger line)
 {
    info->expr = parser_do_expression( p);
    return( _parser_do_assignment_or_expr( p, info, line));
801f8317
 }
 
 
c533990f
 static MulleScionSet  * NS_RETURNS_RETAINED parser_do_set( parser *p, NSUInteger line)
 {
    assignment_or_expr_info   info;
27dc2200
 
c533990f
    init_assignment_or_expr_info( &info, "=", NO);
    parser_do_assignment_or_expr( p, &info, line);
    return( [MulleScionSet newWithRetainedLeftExpression:info.lexpr
                                 retainedRightExpression:info.expr
                                          lineNumber:line]);
 }
 
801f8317
 
c533990f
 static MulleScionObject  * NS_RETURNS_RETAINED parser_do_implicit_set( parser *p, MulleScionExpression * NS_CONSUMED lexpr, NSUInteger line)
2995f4ad
 {
7e71be68
    MulleScionExpression  *expr;
    unsigned char         c;
    char                  *suggestion;
    NSString              *identifier;
27dc2200
 
776920e7
    NSCParameterAssert( [lexpr isFunction] || [lexpr isIndexing] || [lexpr isIdentifier]);
27dc2200
 
c533990f
    c = parser_peek_character( p);
2995f4ad
    if( c != '=')
801f8317
    {
c533990f
       if( [lexpr isIdentifier])
       {
          identifier = [(MulleScionVariable *)  lexpr identifier];
          suggestion = parser_best_match_for_string( p, identifier);
          if( suggestion)
             parser_error( p, "unknown keyword \"%@\" (did you mean \"%s\" ?)", identifier, suggestion);
       }
       parser_error( p, "unknown keyword (maybe you forgot the keyword \"set\" ?)");
801f8317
    }
27dc2200
 
c533990f
    parser_skip_peeked_character( p, '=');
2995f4ad
    parser_skip_whitespace( p);
    expr = parser_do_expression( p);
27dc2200
 
c533990f
    return( [MulleScionSet newWithRetainedLeftExpression:lexpr
                                 retainedRightExpression:expr
                                              lineNumber:line]);
2995f4ad
 }
 
 
 static MulleScionFor  * NS_RETURNS_RETAINED parser_do_for( parser *p, NSUInteger line)
 {
c533990f
    assignment_or_expr_info   info;
27dc2200
 
c533990f
    init_assignment_or_expr_info( &info, "in", NO);
    parser_do_assignment_or_expr( p, &info, line);
    return( [MulleScionFor newWithRetainedLeftExpression:info.lexpr
                                 retainedRightExpression:info.expr
                                              lineNumber:line]);
2995f4ad
 }
 
 
 static MulleScionIf  * NS_RETURNS_RETAINED parser_do_if( parser *p, NSUInteger line)
 {
94c1e4be
    assignment_or_expr_info   info;
7e71be68
    MulleScionExpression      *expr;
c533990f
    int                       is_assignment;
 
    init_assignment_or_expr_info( &info, "=", YES);
    is_assignment =  parser_do_assignment_or_expr( p, &info, line);
 
    expr = info.expr;
    if( is_assignment)
    {
       expr = [MulleScionAssignmentExpression newWithRetainedLeftExpression:info.lexpr
                                                    retainedRightExpression:info.expr
                                                                 lineNumber:line];
    }
27dc2200
 
2995f4ad
    return( [MulleScionIf newWithRetainedExpression:expr
                                      lineNumber:line]);
 }
 
4def952f
 
c533990f
 // while expr || while lexpr = expr
2995f4ad
 static MulleScionWhile  * NS_RETURNS_RETAINED parser_do_while( parser *p, NSUInteger line)
 {
94c1e4be
    assignment_or_expr_info   info;
7e71be68
    MulleScionExpression      *expr;
c533990f
    int                       is_assignment;
27dc2200
 
c533990f
    init_assignment_or_expr_info( &info, "=", YES);
    is_assignment =  parser_do_assignment_or_expr( p, &info, line);
27dc2200
 
c533990f
    expr = info.expr;
    if( is_assignment)
    {
       expr = [MulleScionAssignmentExpression newWithRetainedLeftExpression:info.lexpr
                                                    retainedRightExpression:info.expr
                                                                 lineNumber:line];
    }
27dc2200
 
2995f4ad
    return( [MulleScionWhile newWithRetainedExpression:expr
                                            lineNumber:line]);
 }
 
 
4def952f
 static MulleScionFilter  * NS_RETURNS_RETAINED parser_do_filter( parser *p, NSUInteger line)
 {
    MulleScionExpression   *expr;
fec98eee
    NSMutableArray         *array;
    NSEnumerator           *rover;
    NSString               *keyword;
    NSUInteger             flags;
    MulleScionVariable     *var;
27dc2200
 
4def952f
    expr = parser_do_expression( p);
083dc82f
    if( [expr isFunction])
       parser_error( p, "missing comma after filter identifier");
27dc2200
 
08b5d36d
    if( ! [expr isIdentifier] && ! [expr isPipe]  && ! [expr isMethod])
4def952f
       parser_error( p, "identifier or pipe expected");
27dc2200
 
fec98eee
    flags = FilterOutput|FilterPlaintext;
 
    parser_skip_whitespace( p);
083dc82f
    switch( parser_peek_character( p))
fec98eee
    {
083dc82f
    case ',' :
fec98eee
       parser_next_character( p);
       parser_skip_whitespace( p);
       parser_peek_expected_character( p, '(', "array expected after filter declaration");
083dc82f
 
fec98eee
       // array ? -- allow plaintext and output as keywords
       flags = 0;
       array = parser_do_array_or_arguments( p, NO);
       rover = [array objectEnumerator];
       while( var = [rover nextObject])
       {
          if( [var isIdentifier])
          {
             keyword = [var identifier];
             if( [keyword isEqual:@"plaintext"])
             {
                flags |= FilterPlaintext;  // plaintext ok
                continue;
             }
             if( [keyword isEqual:@"output"])
             {
                flags |= FilterOutput;
                continue;
             }
          }
          parser_error( p, "only plaintext or output are valid");
       }
    }
 
    if( ! flags)
       parser_error( p, "nothing left to filter");
27dc2200
 
4def952f
    return( [MulleScionFilter newWithRetainedExpression:expr
fec98eee
                                                  flags:flags
4def952f
                                             lineNumber:line]);
 }
 
 
08b5d36d
 static MulleScionObject  * NS_RETURNS_RETAINED   parser_do_define( parser *p, NSUInteger line)
 {
c533990f
    MulleScionExpression   *expr;
    NSString               *identifier;
27dc2200
 
c533990f
    identifier = parser_do_identifier( p);
    if( ! identifier)
       parser_error( p, "an identifier after define was expected");
27dc2200
 
801f8317
    if( [identifier hasPrefix:@"MulleScion"] || [identifier hasPrefix:@"__"])
       parser_error( p, "you can't define internal constants");
 
    if( parser_opcode_for_string( p, identifier))
       parser_error( p, "you can't override existing commands");
27dc2200
 
fec98eee
    if( [p->tables.definitionTable objectForKey:identifier])
801f8317
       parser_error( p, "\"%@\" is already defined", identifier);
27dc2200
 
c533990f
    parser_skip_whitespace( p);
    parser_next_expected_character( p, '=', "a '=' was expected after the identifier");
27dc2200
 
c533990f
    expr = parser_do_expression( p);
27dc2200
 
fec98eee
    [p->tables.definitionTable setObject:expr
bdb5913e
                                  forKey:identifier];
08b5d36d
    [expr release];
27dc2200
 
08b5d36d
    return( nil);
 }
 
 
776920e7
 static MulleScionObject  * NS_RETURNS_RETAINED   _parser_do_macro( parser *p, NSUInteger line)
08b5d36d
 {
7e71be68
    MulleScionTemplate  *root;
    MulleScionFunction  *function;
    macro_type          last_type;
    MulleScionObject    *node;
    MulleScionObject    *last;
    MulleScionMacro     *macro;
    NSString            *identifier;
27dc2200
 
2ae60c68
    if( p->inMacro)
       parser_error( p, "no macro definitions in a macro definition.");
27dc2200
 
08b5d36d
    parser_skip_whitespace( p);
    identifier = parser_do_identifier( p);
27dc2200
 
08b5d36d
    parser_skip_whitespace( p);
c533990f
    parser_peek_expected_character( p, '(', "'(' after identifier expected");
08b5d36d
 
fec98eee
    function = [__parser_do_macro( p, identifier) autorelease];
27dc2200
 
ef903235
    identifier = [function identifier];
08b5d36d
    if( [identifier hasPrefix:@"MulleScion"])
       parser_error( p, "you can't define MulleScion macros");
27dc2200
 
fec98eee
    if( [p->tables.macroTable objectForKey:identifier])
08b5d36d
       parser_error( p, "macro %@ is already defined", identifier);
 
c533990f
    // macro %} must be a terminated by %}\n
08b5d36d
    parser_finish_command( p);
27dc2200
 
fec98eee
    // do this once and store in parser
b8c659f4
    if( p->environment & MULLESCION_DUMP_MACROS)
fec98eee
       fprintf( stderr, "defining macro \"%s\"\n", [identifier cString]);
27dc2200
 
08b5d36d
    // now just grab stuff until we hit an endmacro
27dc2200
 
08b5d36d
    root       = [[[MulleScionTemplate alloc] initWithFilename:p->fileName] autorelease];
    last_type  = eof;
    p->inMacro = YES;
 
b8c659f4
    last = nil;
    node = root;
    while( node)
    {
       last = node;
       node = parser_next_object( p, node, &last_type);
    }
08b5d36d
 
    p->inMacro = NO;
801f8317
    if( last_type != command)
       parser_error( p, "endmacro expected", identifier);
27dc2200
 
c533990f
    //
    // take out single trailing linefeed from macro, which is just ugly
    //
    if( last)
    {
       while( last->next_)
          last = last->next_;
27dc2200
 
c533990f
       if( [last isJustALinefeed])
       {
          for( node = root; node->next_ != last; node = node->next_);
27dc2200
 
c533990f
          node->next_ = nil;
          [last release];
       }
    }
27dc2200
 
08b5d36d
    macro = [MulleScionMacro newWithIdentifier:identifier
                                      function:function
                                          body:root
                                      fileName:p->fileName
                                    lineNumber:line];
fec98eee
    [p->tables.macroTable setObject:macro
08b5d36d
                      forKey:identifier];
    [macro autorelease];
c533990f
 
04373918
 #if 0
c533990f
    parser_undo_character( p);
    if( parser_peek_character( p) == '\n')
       parser_undo_character( p);
    parser_undo_character( p);
04373918
 #endif
 
    return( macro);
08b5d36d
 }
 
801f8317
 
776920e7
 static MulleScionObject  * NS_RETURNS_RETAINED   parser_do_macro( parser *p, NSUInteger line)
 {
    MulleScionObject   *obj;
    int                memo;
27dc2200
 
776920e7
    memo            = p->skipComments;
    p->skipComments = 0;
    obj             = _parser_do_macro( p, line);
    p->skipComments = memo;
    return( obj);
 }
 
 
 static MulleScionObject  * NS_RETURNS_RETAINED   _parser_do_verbatim( parser *p, NSUInteger line)
08b5d36d
 {
801f8317
    parser_memo   plaintext_start;
    parser_memo   plaintext_end;
    NSString      *s;
27dc2200
 
801f8317
    parser_finish_command( p);
    parser_memorize( p, &plaintext_start);
08b5d36d
 
801f8317
    if( ! parser_grab_text_until_command( p, "endverbatim"))
       parser_error( p, "no matching endverbatim found");
08b5d36d
 
801f8317
    parser_memorize( p, &plaintext_end);
27dc2200
 
801f8317
    s = parser_get_memorized_retained_string( &plaintext_start, &plaintext_end);
27dc2200
 
801f8317
    // completely forget about endverbatim
    parser_skip_text_until_scion_end( p, '%');
27dc2200
 
c533990f
    parser_undo_character( p);
    parser_undo_character( p);
 
801f8317
    if( ! s)
       return( nil);
 
    return( [MulleScionPlainText newWithRetainedString:s
                                            lineNumber:plaintext_start.lineNumber]);
 }
2995f4ad
 
 
776920e7
 static MulleScionObject  * NS_RETURNS_RETAINED   parser_do_verbatim( parser *p, NSUInteger line)
 {
    MulleScionObject   *obj;
    int                memo;
27dc2200
 
776920e7
    memo            = p->skipComments;
    p->skipComments = 0;
    obj             = _parser_do_verbatim( p, line);
    p->skipComments = memo;
    return( obj);
 }
 
 
801f8317
 static MulleScionObject  * NS_RETURNS_RETAINED   parser_do_endmacro( parser *p, NSUInteger line)
2995f4ad
 {
801f8317
    if( ! p->inMacro)
       parser_error( p, "stray endmacro detected");
    return( nil);
2995f4ad
 }
 
 
d2f30cd2
 static MulleScionObject  * NS_RETURNS_RETAINED   parser_do_print( parser *p, NSUInteger line)
 {
    MulleScionExpression   *expr;
27dc2200
 
d2f30cd2
    parser_skip_peeked_character( p, '{');
    expr = parser_do_expression( p);
27dc2200
 
d2f30cd2
    parser_skip_whitespace( p);
    parser_next_expected_character( p, '}', "closing }} expected");
    parser_next_expected_character( p, '}', "closing }} expected");
27dc2200
 
d2f30cd2
    return( expr);
 }
 
 
 
2995f4ad
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_command( parser *p)
 {
94c1e4be
    NSUInteger             line;
    unsigned char          c;
7e71be68
    MulleScionOpcode       op;
    MulleScionExpression   *expr;
    NSString               *identifier;
c533990f
 
2995f4ad
    line = p->lineNumber;
27dc2200
 
c533990f
    parser_grab_text_until_identifier_end( p);
    identifier = parser_get_string( p);
2995f4ad
    parser_skip_whitespace( p);
27dc2200
 
c533990f
    if( identifier)
    {
       op = parser_opcode_for_string( p, identifier);
 
2ae60c68
    //if( p->inMacro && op != EndmacroOpcode)
    //   parser_error( p, "no commands in macros");
27dc2200
 
ef903235
       switch( (int) op)
c533990f
       {
ef903235
       case BlockOpcode    : return( parser_do_block( p, line));
       case DefineOpcode   : return( parser_do_define( p, line));
       case ElseOpcode     : return( [MulleScionElse newWithLineNumber:line]);
       case ElseforOpcode  : return( [MulleScionElseFor newWithLineNumber:line]);
       case EndblockOpcode : return( [MulleScionEndBlock newWithLineNumber:line]);
       case EndfilterOpcode: return( [MulleScionEndFilter newWithLineNumber:line]);
       case EndforOpcode   : return( [MulleScionEndFor newWithLineNumber:line]);
       case EndifOpcode    : return( [MulleScionEndIf newWithLineNumber:line]);
       case EndmacroOpcode : return( parser_do_endmacro( p, line));
       case EndverbatimOpcode : parser_error( p, "stray endverbatim command");
       case EndwhileOpcode : return( [MulleScionEndWhile newWithLineNumber:line]);
       case ExtendsOpcode  : return( parser_do_extends( p));
       case FilterOpcode   : return( parser_do_filter( p, line));
       case ForOpcode      : return( parser_do_for( p, line));
       case IfOpcode       : return( parser_do_if( p, line));
       case IncludesOpcode : return( parser_do_includes( p, YES));
       case LogOpcode      : return( parser_do_log( p, line));
       case MacroOpcode    : return( parser_do_macro( p, line));
       case RequiresOpcode : return( parser_do_requires( p, line));
       case SetOpcode      : return( parser_do_set( p, line));
       case VerbatimOpcode : return( parser_do_verbatim( p, line));
       case WhileOpcode    : return( parser_do_while( p, line));
c533990f
       }
27dc2200
 
c533990f
       //
       // otherwise an implicit set or macro call
       // dial back and parse expression properly, somewhat wasted effort...
       //
       parser_recall( p, &p->memo);
       expr = (MulleScionIdentifierExpression *) parser_do_expression_or_macro( p);
       if( expr)
       {
          if( p->wasMacroCall)
b8c659f4
          {
             p->wasMacroCall = NO;  // reset now
c533990f
             return( expr);
b8c659f4
          }
448f94fd
          //if( [expr isFunction])  // not that great an idea, produces surprising
          //   return( expr);       // output as a side effect
c533990f
          return( parser_do_implicit_set( p, expr, line));
       }
       return( nil);
2995f4ad
    }
27dc2200
 
c533990f
    c  = parser_peek_character( p);
    if( c == '[')
       return( parser_do_method_call( p, line));
27dc2200
 
60f90a83
    // handle inline print commands
d2f30cd2
    if( c == '{')
    {
       parser_skip_peeked_character( p, '{');
       c  = parser_peek_character( p);
27dc2200
 
d2f30cd2
       if( c == '{')
          return( parser_do_print( p, line));
 
       parser_error( p, "a lonely '{' was found, when a command was expected");
    }
27dc2200
 
d2f30cd2
       // hmmm....
60f90a83
    parser_error( p, "an unexpected character '%c' at the beginning of a command was found", c);
c533990f
    return( nil);
2995f4ad
 }
 
 
c1dca06f
 //
 // one or more semicolon separated commands
 //
c533990f
 static MulleScionObject * NS_RETURNS_RETAINED  _parser_do_command_or_nothing( parser *p)
c1dca06f
 {
c533990f
    MulleScionObject   *expr;
c1dca06f
    unsigned char      c;
27dc2200
 
c533990f
    parser_memorize( p, &p->memo_interesting);
 
    // skip semicolons
    for( ;;)
    {
       parser_skip_whitespace( p);
       c = parser_peek_character( p);
       if( c != ';')
          break;
 
       parser_skip_peeked_character( p, ';');
    }
 
c1dca06f
    //
    // allow empty commands, i don't like parser_peek2_character
    // but couldn't think of something better
    //
b8c659f4
    if( parser_peek_character( p) == '{')
    {
       if( parser_peek2_character( p) == '%')
          parser_error( p, "command nested within command");
       if( parser_peek2_character( p) == '#')
          parser_error( p, "comment nested within command");
    }
27dc2200
 
c1dca06f
    if( parser_peek_character( p) == '%' && parser_peek2_character( p) == '}')
       return( nil);
27dc2200
 
c1dca06f
    parser_skip_whitespace( p);
c533990f
    expr = parser_do_command( p);
27dc2200
 
b8c659f4
    if( p->environment & MULLESCION_DUMP_COMMANDS)
       fprintf( stderr, "%s\n", [[expr dumpDescription] cString]);
c533990f
    return( expr);
 }
c1dca06f
 
c533990f
 
776920e7
 static MulleScionObject * NS_RETURNS_RETAINED  _parser_do_commands( parser *p)
c533990f
 {
    MulleScionObject   *first;
    MulleScionObject   *expr;
    MulleScionObject   *next;
27dc2200
 
c533990f
    expr = _parser_do_command_or_nothing( p);
    if( ! expr)
       return( expr);
 
    // commands like includes, extends, block, endblock can not be in multiline
b8c659f4
    // statements (but requires can, but really shouldn't)
27dc2200
 
c533990f
    if( [expr snarfsScion])
       return( expr);
27dc2200
 
c533990f
    first = expr;
    while( next = _parser_do_command_or_nothing( p))
    {
       expr->next_ = next;
b8c659f4
 
       // dial down the macro chain, if any
       while( next->next_)
          next = next->next_;
27dc2200
 
b8c659f4
       expr = next;
c533990f
    }
    return( first);
c1dca06f
 }
 
 
776920e7
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_commands( parser *p)
 {
    MulleScionObject   *obj;
 
    assert( p->skipComments <= 0);
27dc2200
 
776920e7
    p->skipComments++;
    obj = _parser_do_commands( p);
    p->skipComments--;
    return( obj);
 }
 
 
2995f4ad
 static MulleScionObject * NS_RETURNS_RETAINED  parser_do_block_break_on_extends_skip_others( parser *p)
 {
    NSString   *s;
27dc2200
 
2995f4ad
    parser_skip_whitespace( p);
    s = parser_do_identifier( p);
27dc2200
 
801f8317
    switch( parser_opcode_for_string( p, s))
2995f4ad
    {
08b5d36d
    case BlockOpcode :
       parser_do_whole_block_to_block_table( p);
       break;
27dc2200
 
08b5d36d
    case ExtendsOpcode :
       return( parser_do_extends( p));
27dc2200
 
08b5d36d
    default :
       break;
2995f4ad
    }
    return( nil);
 }
 
 
801f8317
 static MulleScionObject  * NS_RETURNS_RETAINED _parser_next_object_after_extend( parser *p, macro_type *last_type)
2995f4ad
 {
08b5d36d
    macro_type         type;
    MulleScionObject   *obj;
27dc2200
 
08b5d36d
    obj = nil;
 retry:
fc1b9d06
    type = parser_grab_text_until_scion_start( p);
c533990f
    parser_next_character( p);  // laissez faire
08b5d36d
    parser_next_character( p);
27dc2200
 
08b5d36d
    *last_type = type;
ef903235
    switch( (int) type)
2995f4ad
    {
801f8317
    case eof :
       return( nil);
27dc2200
 
801f8317
    case comment :
       parser_finish_comment( p);
       goto retry;
27dc2200
 
801f8317
    case expression :
       parser_finish_expression( p);
       break;
27dc2200
 
801f8317
    case command :
       obj = parser_do_block_break_on_extends_skip_others( p);
       break;
2995f4ad
    }
27dc2200
 
08b5d36d
    return( obj);
2995f4ad
 }
 
 
801f8317
 static MulleScionObject  *_parser_next_object( parser *p,  MulleScionObject *owner, macro_type *last_type)
2995f4ad
 {
7e71be68
    MulleScionObject    *next;
    parser_memo         plaintext_start;
    parser_memo         plaintext_end;
    NSString            *s;
    macro_type          type;
801f8317
 
2995f4ad
 retry:
801f8317
    next     = nil;
2995f4ad
    p->first = nil;
27dc2200
 
2995f4ad
    parser_memorize( p, &plaintext_start);
fc1b9d06
    type = parser_grab_text_until_scion_start( p);
2995f4ad
    parser_memorize( p, &plaintext_end);
    parser_next_character( p);
    parser_next_character( p);
27dc2200
 
2995f4ad
    if( type == comment || type == command)
       parser_adjust_memo_to_end_of_previous_line( p, &plaintext_end);
27dc2200
 
2995f4ad
    s = parser_get_memorized_retained_string( &plaintext_start, &plaintext_end);
    if( s)
    {
       next     = [MulleScionPlainText newWithRetainedString:s
                                                  lineNumber:plaintext_start.lineNumber];
       p->first = next;
b8c659f4
 
       [owner appendRetainedObject:next];
       owner    = [next tail];
2995f4ad
    }
27dc2200
 
2995f4ad
    *last_type = type;
27dc2200
 
ef903235
    switch( (int) type)
2995f4ad
    {
801f8317
    case eof :
       return( nil);
27dc2200
 
801f8317
    case comment :
       parser_finish_comment( p);
       goto retry;
27dc2200
 
801f8317
    case expression :
       parser_skip_whitespace( p);
c533990f
       parser_memorize( p, &p->memo_interesting);
801f8317
       next = parser_do_expression_or_macro( p);
       parser_finish_expression( p);
27dc2200
 
fb5e6269
       if( p->environment & MULLESCION_DUMP_EXPRESSIONS)
          fprintf( stderr, "%s\n", [[next dumpDescription] cString]);
801f8317
       break;
27dc2200
 
801f8317
    case command :
       parser_skip_whitespace( p);
c1dca06f
       next = parser_do_commands( p);
c533990f
       if( ! [next snarfsScion])
801f8317
          parser_finish_command( p);
2ae60c68
       if( ! next && p->inMacro)
          return( nil);  // endmacro ...
04373918
       if( [next isMacro])
          next = nil;
801f8317
       break;
2995f4ad
    }
27dc2200
 
2995f4ad
    //
c1dca06f
    // the problem, is that the analyzer thinks with NS_CONSUMED that the
2995f4ad
    // object is dead, while we know it's alive
    //
08b5d36d
    if( next)
    {
       if( ! p->first)
          p->first = next;
27dc2200
 
b8c659f4
       [owner appendRetainedObject:next];  // analyzer mistake
       owner = [next tail];
08b5d36d
    }
2995f4ad
    return( owner);
 }
 
 
99fb8c65
 # pragma mark -
 # pragma mark External Interface (API)
 
08b5d36d
 - (void) parseData:(NSData *) data
fec98eee
      intoRootObject:(MulleScionObject *) root
             tables:(MulleScionParserTables *) tables
       ignoreErrors:(BOOL) ignoreErrors
2995f4ad
 {
08b5d36d
    parser              parser;
    macro_type          last_type;
 
    parser_init( &parser, (void *) [data_ bytes], [data_ length]);
    parser_set_filename( &parser, fileName_);
fec98eee
    parser_set_blocks_table( &parser, tables->blockTable);
    parser_set_definitions_table( &parser, tables->definitionTable);
    parser_set_macro_table( &parser, tables->macroTable);
    parser_set_dependency_table( &parser, tables->dependencyTable);
bdb5913e
    parser_set_error_callback( &parser, self, @selector( parserError:));
    parser_set_warning_callback( &parser, self, @selector( parserWarning:));
fec98eee
 
2523ba40
    //
    // this make it possible to have scion templates as executable unix scripts
    // lets do this also on includes, because otherwise the output of the dox
    // looks funny. Well because they include it verbatim, they still look funny
    //   if( ! [self parent])
fec98eee
    //
b8c659f4
    if( ! (parser.environment & MULLESCION_NO_HASHBANG))
2523ba40
       parser_skip_initial_hashbang_line_if_present( &parser);
fec98eee
 
    // useful to just run dependency, even if syntactically garbage
    if( ignoreErrors)
    {
       volatile MulleScionObject    *node;
27dc2200
 
b8c659f4
       // reset environment to not talkative
       parser.environment &= MULLESCION_NO_HASHBANG | MULLESCION_VERBATIM_INCLUDE_HASHBANG;
27dc2200
 
fec98eee
       last_type = eof;
       node      = root;
27dc2200
 
fec98eee
 retry:
 NS_DURING
       while( node)
65526c3f
          node = parser_next_object( &parser, (id) node, &last_type);
fec98eee
 NS_HANDLER
       // skip to next %}
       parser.skipComments = 0;
       if( parser_skip_text_until_scion_end( &parser, '%') != eof)
       {
          while( node->next_)
             node = node->next_;
27dc2200
 
fec98eee
          goto retry;
       }
 NS_ENDHANDLER
       return;
    }
    else
    {
       MulleScionObject    *node;
27dc2200
 
fec98eee
       last_type = eof;
       for( node = root; node; node = parser_next_object( &parser, node, &last_type));
    }
2995f4ad
 }
 
 
fec98eee
 - (MulleScionTemplate *) templateParsedWithTables:(MulleScionParserTables *) tables
2995f4ad
 {
    MulleScionTemplate  *root;
27dc2200
 
801f8317
    root = [[[MulleScionTemplate alloc] initWithFilename:[fileName_ lastPathComponent]] autorelease];
27dc2200
 
08b5d36d
    [self parseData:data_
     intoRootObject:root
fec98eee
             tables:tables
       ignoreErrors:NO];
27dc2200
 
2995f4ad
    return( root);
 }
 
801f8317
 
08b5d36d
 - (MulleScionTemplate *) templateWithContentsOfFile:(NSString *) fileName
fec98eee
                                              tables:(MulleScionParserTables *) tables
                                           converter:(SEL) converterSel
08b5d36d
 {
    MulleScionParser    *parser;
    MulleScionTemplate  *template;
    NSData              *data;
    NSString            *dir;
801f8317
    NSString            *path;
27dc2200
 
801f8317
 retry:
    path = fileName;
    data = [NSData dataWithContentsOfFile:path];
08b5d36d
    if( ! data)
    {
801f8317
       dir  = [fileName_ stringByDeletingLastPathComponent];
       path = [dir stringByAppendingPathComponent:path];
27dc2200
 
801f8317
       data = [NSData dataWithContentsOfFile:path];
08b5d36d
       if( ! data)
801f8317
       {
          if( ! [[fileName pathExtension] length])
          {
             fileName = [fileName stringByAppendingPathExtension:@"scion"];
             goto retry;
          }
08b5d36d
          return( nil);
801f8317
       }
08b5d36d
    }
27dc2200
 
fec98eee
    if( getenv( "MULLESCION_DUMP_FILEPATHS"))
       fprintf( stderr, "parsing \"%s\"\n", [path fileSystemRepresentation]);
27dc2200
 
fec98eee
    /* run data through converter if needed */
    if( converterSel)
       data = [data performSelector:converterSel
                         withObject:nil];
27dc2200
 
fec98eee
    parser = [[[MulleScionParser alloc] initWithData:data
                                            fileName:path] autorelease];
    template = [parser templateParsedWithTables:tables];
801f8317
    return( template);
08b5d36d
 }
 
2995f4ad
 @end