From 15ac04f50532a675ef609a0f04701755e8ad0f2c Mon Sep 17 00:00:00 2001
From: zawz <zawz@hotmail.fr>
Date: Fri, 21 May 2021 16:36:24 +0200
Subject: [PATCH] implement delayed heredocument parsing

---
 include/parse.hpp    |  21 ++-
 include/struc.hpp    |  76 +++++---
 src/exec.cpp         |   2 +-
 src/generate.cpp     |  69 +++++---
 src/parse.cpp        | 402 +++++++++++++++++++------------------------
 src/struc_helper.cpp |   3 +-
 6 files changed, 292 insertions(+), 281 deletions(-)

diff --git a/include/parse.hpp b/include/parse.hpp
index 95575d4..9a49959 100644
--- a/include/parse.hpp
+++ b/include/parse.hpp
@@ -6,6 +6,7 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <tuple>
 
 #define SPACES          " \t"
 #define SEPARATORS      " \t\n"
@@ -33,9 +34,13 @@
 
 // structs
 
-void parse_error(std::string const& message, parse_context& ctx);
+struct list_parse_options {
+  char end_char=0;
+  bool word_mode=false;
+  std::vector<std::string> end_words={};
+  const char* expecting=NULL;
+};
 
-void parse_error(std::string const& message, parse_context& ctx, uint64_t i);
 // globals
 
 extern const std::vector<std::string> posix_cmdvar;
@@ -48,12 +53,17 @@ std::pair<shmain*, parse_context> parse_text(std::string const& in, std::string
 inline std::pair<shmain*, parse_context> parse(std::string const& file) { return parse_text(import_file(file), file); }
 
 // tools
+
 parse_context make_context(std::string const& in, std::string const& filename="", bool bash=false);
 parse_context make_context(parse_context ctx, std::string const& in="", std::string const& filename="", bool bash=false);
 parse_context make_context(parse_context ctx, uint64_t i);
 parse_context operator+(parse_context ctx, int64_t a);
 parse_context operator-(parse_context ctx, int64_t a);
 
+// error handlers
+void parse_error(std::string const& message, parse_context& ctx);
+void parse_error(std::string const& message, parse_context& ctx, uint64_t i);
+
 // ** unit parsers ** //
 
 /* util parsers */
@@ -76,9 +86,10 @@ inline uint32_t skip_unread(parse_context const& ct) {
 }
 
 // list
-std::pair<list*, parse_context> parse_list_until(parse_context ct, char end_c, const char* expecting=NULL);
-std::pair<list*, parse_context> parse_list_until(parse_context ct, std::string const& end_word);
-std::tuple<list*, parse_context, std::string> parse_list_until(parse_context ct, std::vector<std::string> const& end_words, const char* expecting=NULL);
+// std::pair<list*, parse_context> parse_list_until(parse_context ct, char end_c, const char* expecting=NULL);
+// std::pair<list*, parse_context> parse_list_until(parse_context ct, std::string const& end_word);
+// std::tuple<list*, parse_context, std::string> parse_list_until(parse_context ct, std::vector<std::string> const& end_words, const char* expecting=NULL);
+std::tuple<list*, parse_context, std::string> parse_list_until(parse_context ct, list_parse_options opts={});
 // name
 std::pair<variable*, parse_context> parse_var(parse_context ct, bool specialvars=true, bool array=false);
 
diff --git a/include/struc.hpp b/include/struc.hpp
index 8669fa1..2bb87eb 100644
--- a/include/struc.hpp
+++ b/include/struc.hpp
@@ -62,6 +62,21 @@ subarg: can be one of
 
 */
 
+// pre-definitions
+
+#define AND_OP false
+#define OR_OP  true
+
+class condlist;
+class block;
+class pipeline;
+class arg;
+class subarg;
+class cmd;
+class redirect;
+
+// structs
+
 struct parse_context {
   const char* data=NULL;
   uint64_t size=0;
@@ -73,9 +88,16 @@ struct parse_context {
   const char* here_doc="";
   const char operator[](uint64_t a) { return data[a]; }
   bool has_errored=false;
+  redirect* here_document=nullptr;
+  char* here_delimitor=NULL;
+};
+
+struct generate_context {
+  arg* here_document=nullptr;
 };
 
 // exceptions
+
 class format_error : public std::exception
 {
 public:
@@ -97,20 +119,8 @@ private:
   std::string sdat;
 };
 
-
 // objects
 
-
-#define AND_OP false
-#define OR_OP  true
-
-class condlist;
-class block;
-class pipeline;
-class arg;
-class subarg;
-class cmd;
-
 // type pack of condlist
 typedef std::vector<arg*> arglist_t;
 
@@ -229,15 +239,17 @@ public:
 class redirect : public _obj
 {
 public:
-  redirect(std::string strop="") { type=_obj::_redirect; op=strop; target=nullptr; }
-  redirect(arg* in) { type=_obj::_redirect; target=in; }
-  redirect(std::string strop, arg* in) { type=_obj::_redirect; op=strop; target=in; }
+  redirect(std::string strop="") { type=_obj::_redirect; op=strop; target=nullptr; here_document=nullptr; }
+  redirect(arg* in) { type=_obj::_redirect; target=in; here_document=nullptr; }
+  redirect(std::string strop, arg* in) { type=_obj::_redirect; op=strop; target=in; here_document=nullptr; }
+  redirect(std::string strop, arg* in, arg* doc) { type=_obj::_redirect; op=strop; target=in; here_document=doc; }
   ~redirect() { if(target != nullptr) delete target; }
 
   std::string generate(int ind);
 
   std::string op;
   arg* target;
+  arg* here_document;
 };
 
 // Meta block
@@ -252,9 +264,9 @@ public:
   // subshell: return the containing cmd, if it is a single command
   cmd* single_cmd();
 
-  std::string generate_redirs(int ind, std::string const& _str);
+  std::string generate_redirs(int ind, std::string const& _str, generate_context* ctx);
 
-  virtual std::string generate(int ind)=0;
+  virtual std::string generate(int ind, generate_context* ctx)=0;
 };
 
 // PL
@@ -269,7 +281,8 @@ public:
 
   bool negated; // negated return value (! at start)
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); };
 };
 
 // CL
@@ -370,7 +383,8 @@ public:
 
   arglist* args;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 class shmain : public block
@@ -388,7 +402,8 @@ public:
   list* lst;
 
   std::string generate(bool print_shebang=true, int ind=0);
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 class subshell : public block
@@ -404,7 +419,8 @@ public:
 
   list* lst;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 class brace : public block
@@ -419,7 +435,8 @@ public:
 
   list* lst;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 class function : public block
@@ -433,7 +450,8 @@ public:
   std::string name;
   list* lst;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 class case_block : public block
@@ -453,7 +471,8 @@ public:
   arg* carg;
   std::vector< std::pair<std::vector<arg*>, list*> > cases;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 class if_block : public block
@@ -473,7 +492,8 @@ public:
 
   list* else_lst;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 class for_block : public block
@@ -491,7 +511,8 @@ public:
   arglist* iter;
   list* ops;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 class while_block : public block
@@ -508,7 +529,8 @@ public:
   list* cond;
   list* ops;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 // Subarg subtypes //
diff --git a/src/exec.cpp b/src/exec.cpp
index 53eb6ad..fe89834 100644
--- a/src/exec.cpp
+++ b/src/exec.cpp
@@ -152,7 +152,7 @@ void parse_exec(FILE* fd, parse_context ctx)
     ctx=pp.second;
     if(ctx.has_errored)
     {
-      parse_list_until(ctx, 0);
+      parse_list_until(ctx);
       throw std::runtime_error("Aborting due to previous errors");
     }
     t_lst->add(pp.first);
diff --git a/src/generate.cpp b/src/generate.cpp
index 6139c15..41071b2 100644
--- a/src/generate.cpp
+++ b/src/generate.cpp
@@ -44,7 +44,7 @@ std::string arglist::generate(int ind)
   return ret;
 }
 
-std::string pipeline::generate(int ind)
+std::string pipeline::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -53,11 +53,11 @@ std::string pipeline::generate(int ind)
 
   if(negated)
     ret += "! ";
-  ret += cmds[0]->generate(ind);
+  ret += cmds[0]->generate(ind, ctx);
   for(uint32_t i=1 ; i<cmds.size() ; i++)
   {
     ret += opt_minify ? "|" : " | " ;
-    ret += cmds[i]->generate(ind);
+    ret += cmds[i]->generate(ind, ctx);
   }
 
   return ret;
@@ -68,18 +68,27 @@ std::string condlist::generate(int ind)
   std::string ret;
   if(pls.size() <= 0)
     return "";
-  ret += pls[0]->generate(ind);
+  generate_context ctx;
+  ret += pls[0]->generate(ind, &ctx);
   for(uint32_t i=0 ; i<pls.size()-1 ; i++)
   {
     if(or_ops[i])
       ret += opt_minify ? "||" : " || ";
     else
       ret += opt_minify ? "&&" : " && ";
-    ret += pls[i+1]->generate(ind);
+    ret += pls[i+1]->generate(ind, &ctx);
   }
   if(ret=="")
     return "";
-  if(parallel)
+  if(ctx.here_document != nullptr)
+  {
+    if(parallel)
+      ret += '&';
+    ret += '\n';
+    ret += ctx.here_document->generate(0);
+    ret += '\n';
+  }
+  else if(parallel)
   {
     ret += opt_minify ? "&" : " &\n";
   }
@@ -123,12 +132,18 @@ std::string redirect::generate(int ind)
 
 // BLOCK
 
-std::string block::generate_redirs(int ind, std::string const& _str)
+std::string block::generate_redirs(int ind, std::string const& _str, generate_context* ctx=nullptr)
 {
   std::string ret=" ";
   bool previous_isnt_num = _str.size()>0 && !is_num(_str[_str.size()-1]);
   for(auto it: redirs)
   {
+    if(ctx != nullptr && it->here_document != nullptr)
+    {
+      if(ctx->here_document != nullptr)
+        throw std::runtime_error("Unsupported generation of concurrent here documents");
+      ctx->here_document = it->here_document;
+    }
     std::string _r = it->generate(0);
     if(opt_minify && _r.size() > 0 && !is_num(_r[0]) && previous_isnt_num)
       ret.pop_back(); // remove one space if possible
@@ -139,7 +154,7 @@ std::string block::generate_redirs(int ind, std::string const& _str)
   return ret;
 }
 
-std::string if_block::generate(int ind)
+std::string if_block::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -169,11 +184,11 @@ std::string if_block::generate(int ind)
 
   ret += indented("fi", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string for_block::generate(int ind)
+std::string for_block::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -187,11 +202,11 @@ std::string for_block::generate(int ind)
 
   if(opt_minify && ret.size()>1 && !is_alpha(ret[ret.size()-2]))
     ret.pop_back();
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string while_block::generate(int ind)
+std::string while_block::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -207,11 +222,11 @@ std::string while_block::generate(int ind)
 
   if(opt_minify && ret.size()>1 && !is_alpha(ret[ret.size()-2]))
     ret.pop_back();
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string subshell::generate(int ind)
+std::string subshell::generate(int ind, generate_context* ctx)
 {
   std::string ret;
   // open subshell
@@ -224,11 +239,11 @@ std::string subshell::generate(int ind)
   // close subshell
   ret += indented(")", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string shmain::generate(int ind)
+std::string shmain::generate(int ind, generate_context* ctx)
 {
   return this->generate(false, ind);
 }
@@ -241,11 +256,10 @@ std::string shmain::generate(bool print_shebang, int ind)
   if( opt_minify && ret[ret.size()-1] == '\n')
     ret.pop_back();
 
-  ret += generate_redirs(ind, ret);
   return ret;
 }
 
-std::string brace::generate(int ind)
+std::string brace::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -253,11 +267,11 @@ std::string brace::generate(int ind)
   ret += lst->generate(ind+1);
   ret += indented("}", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string function::generate(int ind)
+std::string function::generate(int ind, generate_context* ctx)
 {
   std::string ret;
   // function definition
@@ -268,11 +282,11 @@ std::string function::generate(int ind)
   ret += lst->generate(ind+1);
   ret += indented("}", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string case_block::generate(int ind)
+std::string case_block::generate(int ind, generate_context* ctx)
 {
   std::string ret;
   ret += "case " + carg->generate(ind) + " in\n";
@@ -307,14 +321,15 @@ std::string case_block::generate(int ind)
   ind--;
   ret += indented("esac", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string cmd::generate(int ind)
+std::string cmd::generate(int ind, generate_context* ctx)
 {
   std::string ret;
-  // var assigns
+
+  // is a varassign cmd
   if(is_cmdvar)
   {
     ret += args->generate(ind) + ' ';
@@ -330,6 +345,7 @@ std::string cmd::generate(int ind)
     return ret;
   }
 
+  // pre-cmd var assigns
   for(auto it: var_assigns)
   {
     if(it.first != nullptr)
@@ -339,6 +355,7 @@ std::string cmd::generate(int ind)
     ret += ' ';
   }
 
+  // cmd itself
   if(args!=nullptr && args->size()>0)
   {
     // command
@@ -353,7 +370,7 @@ std::string cmd::generate(int ind)
       ret.pop_back();
   }
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
diff --git a/src/parse.cpp b/src/parse.cpp
index 5521550..fc848d1 100644
--- a/src/parse.cpp
+++ b/src/parse.cpp
@@ -25,7 +25,6 @@ const std::vector<std::string> out_reserved_words = { "then", "else", "fi", "esa
 
 // stuff
 
-
 std::string unexpected_token(char c)
 {
   std::string print;
@@ -442,9 +441,9 @@ parse_context do_one_subarg_step(arg* ret, parse_context ctx, uint32_t& j, bool
     // get subshell
     parse_context newct = ctx;
     ctx.size=k;
-    auto r=parse_list_until(newct, 0);
-    ret->add(new subshell_subarg(new subshell(r.first), is_quoted));
-    ctx = r.second;
+    auto r=parse_list_until(newct);
+    ret->add(new subshell_subarg(new subshell(std::get<0>(r)), is_quoted));
+    ctx = std::get<1>(r);
     ctx.i++;
     j = ctx.i;
   }
@@ -617,6 +616,35 @@ std::pair<arg*, parse_context> parse_arg(parse_context ctx, const char* end, con
   return std::make_pair(ret, ctx);
 }
 
+parse_context parse_heredocument(parse_context ctx)
+{
+  if(ctx.here_document == nullptr)
+    return ctx;
+
+  uint32_t j=ctx.i;
+  char* tc=NULL;
+  std::string delimitor=ctx.here_delimitor;
+  tc = (char*) strstr(ctx.data+ctx.i, std::string("\n"+delimitor+"\n").c_str()); // find delimitor
+  if(tc!=NULL) // delimitor was found
+  {
+    ctx.i = (tc-ctx.data)+delimitor.size()+1;
+  }
+  else
+  {
+    ctx.i = ctx.size;
+  }
+  // std::string tmpparse=std::string(ctx.data+j, ctx.i-j);
+  auto pval = parse_arg({ .data=ctx.data, .size=ctx.i, .i=j, .bash=ctx.bash} , NULL);
+  ctx.here_document->here_document = pval.first;
+
+  //
+  ctx.here_document=nullptr;
+  free(ctx.here_delimitor);
+  ctx.here_delimitor=NULL;
+
+  return ctx;
+}
+
 std::pair<redirect*, parse_context> parse_redirect(parse_context ctx)
 {
   bool is_redirect=false;
@@ -717,14 +745,20 @@ std::pair<redirect*, parse_context> parse_redirect(parse_context ctx)
         ctx.i = skip_chars(ctx, SPACES);
         if(ret->op == "<<")
         {
+          if(ctx.here_document != nullptr)
+          {
+            parse_error("unsupported multiple here documents at the same time", ctx);
+            return std::make_pair(ret, ctx);
+          }
+          else
+            ctx.here_document=ret;
+
           auto pa = parse_arg(ctx);
           std::string delimitor = pa.first->string();
-          delete pa.first;
-          pa.first = nullptr;
 
           if(delimitor == "")
           {
-            parse_error("Non-static or empty text input delimitor", ctx);
+            parse_error("non-static or empty here document delimitor", ctx);
           }
 
           if(delimitor.find('"') != std::string::npos || delimitor.find('\'') != std::string::npos || delimitor.find('\\') != std::string::npos)
@@ -732,34 +766,11 @@ std::pair<redirect*, parse_context> parse_redirect(parse_context ctx)
             delimitor = ztd::sh("echo "+delimitor); // shell resolve the delimitor
             delimitor.pop_back(); // remove \n
           }
-
+          ret->target = pa.first;
           ctx = pa.second;
-          ctx.i = skip_chars(ctx, SPACES);  // skip spaces
-
-          if(ctx[ctx.i] == '#') // skip comment
-            ctx.i = skip_until(ctx, "\n"); //skip to endline
-          if(ctx[ctx.i] != '\n') // has another arg
-          {
-            parse_error("Additionnal argument after text input delimitor", ctx);
-            return std::make_pair(ret, ctx);
-          }
-
-          ctx.i++;
-          uint32_t j=ctx.i;
-          char* tc=NULL;
-          tc = (char*) strstr(ctx.data+ctx.i, std::string("\n"+delimitor+"\n").c_str()); // find delimitor
-          if(tc!=NULL) // delimitor was found
-          {
-            ctx.i = (tc-ctx.data)+delimitor.size()+1;
-          }
-          else
-          {
-            ctx.i = ctx.size;
-          }
-          std::string tmpparse=std::string(ctx.data+j, ctx.i-j);
-          auto pval = parse_arg({ .data=tmpparse.c_str(), .size=tmpparse.size(), .i=0, .bash=ctx.bash}, NULL);
-          ret->target = pval.first;
-          ret->target->insert(0, delimitor+"\n");
+          // copy delimitor
+          ctx.here_delimitor = (char*) malloc(delimitor.length()+1);
+          strcpy(ctx.here_delimitor, delimitor.c_str());
         }
         else
         {
@@ -1017,67 +1028,7 @@ std::pair<condlist*, parse_context> parse_condlist(parse_context ctx)
   return std::make_pair(ret, ctx);
 }
 
-std::pair<list*, parse_context> parse_list_until(parse_context ctx, char end_c, const char* expecting)
-{
-  list* ret = new list;
-  ctx.i=skip_unread(ctx);
-
-#ifndef NO_PARSE_CATCH
-  try
-  {
-#endif
-;
-    const char* old_expect=ctx.expecting;
-    if(ctx.expecting!=NULL)
-      ctx.expecting = expecting;
-
-    while(ctx[ctx.i] != end_c)
-    {
-      auto pp=parse_condlist(ctx);
-      ret->add(pp.first);
-      ctx=pp.second;
-
-      if(ctx.i < ctx.size)
-      {
-        if(ctx[ctx.i] == end_c) // end char, stop here
-          break;
-        else if(ctx[ctx.i] == '#')
-          ; // skip here
-        else if(is_in(ctx[ctx.i], COMMAND_SEPARATOR))
-          ctx.i++; // skip on next char
-        else if(is_in(ctx[ctx.i], CONTROL_END))
-        {
-          parse_error( unexpected_token(ctx[ctx.i]), ctx);
-          return std::make_pair(ret, ctx);
-        }
-
-        ctx.i = skip_unread(ctx);
-      }
-
-      if(ctx.i>=ctx.size)
-      {
-        if(end_c != 0)
-        {
-          parse_error(strf("Expecting '%s'", expecting), ctx);
-          return std::make_pair(ret, ctx);
-        }
-        else
-          break;
-      }
-    }
-    ctx.expecting = old_expect;
-#ifndef NO_PARSE_CATCH
-  }
-  catch(format_error& e)
-  {
-    delete ret;
-    throw e;
-  }
-#endif
-  return std::make_pair(ret, ctx);
-}
-
-std::pair<list*, parse_context> parse_list_until(parse_context ctx, std::string const& end_word)
+std::tuple<list*, parse_context, std::string> parse_list_until(parse_context ctx, list_parse_options opts)
 {
   list* ret = new list;
   ctx.i=skip_unread(ctx);
@@ -1088,114 +1039,114 @@ std::pair<list*, parse_context> parse_list_until(parse_context ctx, std::string
   {
 #endif
 ;
+
+    char& end_c = opts.end_char;
+    std::vector<std::string>& end_words = opts.end_words;
+
     const char* old_expect=ctx.expecting;
 
-    ctx.expecting=end_word.c_str();
-
-    while(true)
-    {
-      // check word
-      auto wp=get_word(ctx, ARG_END);
-      if(wp.first == end_word)
-      {
-        ctx.i=wp.second;
-        break;
-      }
-      // do a parse
-      auto pp=parse_condlist(ctx);
-      ret->add(pp.first);
-      ctx=pp.second;
-      if(ctx.i<ctx.size)
-      {
-        if(ctx[ctx.i] == '#')
-          ; // skip here
-        else if(is_in(ctx[ctx.i], COMMAND_SEPARATOR))
-          ctx.i++;
-        else if(is_in(ctx[ctx.i], CONTROL_END))
-        {
-          parse_error(unexpected_token(ctx[ctx.i]), ctx);
-          return std::make_pair(ret, ctx);
-        }
-
-        ctx.i = skip_unread(ctx);
-      }
-      // word wasn't found
-      if(ctx.i>=ctx.size)
-      {
-        parse_error(strf("Expecting '%s'", ctx.expecting), ctx);
-        return std::make_pair(ret, ctx);
-      }
-    }
-    ctx.expecting=old_expect;
-#ifndef NO_PARSE_CATCH
-  }
-  catch(format_error& e)
-  {
-    delete ret;
-    throw e;
-  }
-#endif
-  return std::make_pair(ret, ctx);
-}
-
-
-std::tuple<list*, parse_context, std::string> parse_list_until(parse_context ctx, std::vector<std::string> const& end_words, const char* expecting)
-{
-  list* ret = new list;
-  ctx.i=skip_unread(ctx);
-  std::string found_end_word;
-
-#ifndef NO_PARSE_CATCH
-  try
-  {
-#endif
-;
-    const char* old_expect=ctx.expecting;
-
-    if(expecting!=NULL)
-      ctx.expecting=expecting;
-    else
+    if(opts.expecting!=NULL)
+      ctx.expecting=opts.expecting;
+    else if(opts.word_mode)
       ctx.expecting=end_words[0].c_str();
+    else
+      ctx.expecting=std::string(&end_c, 1).c_str();
 
     bool stop=false;
     while(true)
     {
-      // check words
-      auto wp=get_word(ctx, ARG_END);
-      for(auto it: end_words)
+      if(opts.word_mode)
       {
-        if(it == ";" && ctx[ctx.i] == ';')
+        // check words
+        auto wp=get_word(ctx, ARG_END);
+        for(auto it: end_words)
         {
-          found_end_word=";";
-          ctx.i++;
-          stop=true;
-          break;
+          if(it == ";" && ctx[ctx.i] == ';')
+          {
+            found_end_word=";";
+            ctx.i++;
+            stop=true;
+            break;
+          }
+          if(wp.first == it)
+          {
+            found_end_word=it;
+            ctx.i=wp.second;
+            stop=true;
+            break;
+          }
         }
-        if(wp.first == it)
-        {
-          found_end_word=it;
-          ctx.i=wp.second;
-          stop=true;
+        if(stop)
           break;
-        }
       }
-      if(stop)
+      else if(ctx[ctx.i] == end_c)
+      {
         break;
+      }
       // do a parse
       auto pp=parse_condlist(ctx);
       ret->add(pp.first);
       ctx=pp.second;
-      if(ctx[ctx.i] == '#')
+
+      if(!opts.word_mode && ctx[ctx.i] == end_c)
+        break; // reached end char: stop here
+      else if(ctx[ctx.i] == '\n')
+      {
+        if(ctx.here_document != nullptr)
+          ctx = parse_heredocument(ctx+1);
+        // do here document parse
+      }
+      else if(ctx[ctx.i] == '#')
         ; // skip here
       else if(is_in(ctx[ctx.i], COMMAND_SEPARATOR))
-        ctx.i++; // skip on next
+        ; // skip on next
+      else if(is_in(ctx[ctx.i], CONTROL_END))
+      {
+        // control end: unexpected
+        parse_error( unexpected_token(ctx[ctx.i]), ctx);
+        break;
+      }
+
+      if(ctx.here_document != nullptr)
+      {
+        uint8_t do_twice=2;
+        // case of : cat << EOF ;
+        while(do_twice>0)
+        {
+          if(ctx[ctx.i] == '\n')
+          {
+            ctx = parse_heredocument(ctx+1);
+            break;
+          }
+          else if(ctx[ctx.i] == '#')
+          {
+            ctx.i = skip_until(ctx, "\n"); //skip to endline
+            ctx = parse_heredocument(ctx+1);
+            break;
+          }
+          skip_chars(ctx, SPACES);
+          do_twice--;
+        }
+        // case of : cat << EOF ; ;
+        if(do_twice==0 && is_in(ctx[ctx.i], COMMAND_SEPARATOR))
+          parse_error( unexpected_token(ctx[ctx.i]), ctx);
+      }
+
+      if(is_in(ctx[ctx.i], COMMAND_SEPARATOR))
+        ctx.i++;
 
       ctx.i = skip_unread(ctx);
+
       // word wasn't found
       if(ctx.i>=ctx.size)
       {
-        parse_error(strf("Expecting '%s'", ctx.expecting), ctx);
-        return std::make_tuple(ret, ctx, "");
+        if(opts.word_mode || opts.end_char != 0)
+        {
+          parse_error(strf("Expecting '%s'", ctx.expecting), ctx);
+          return std::make_tuple(ret, ctx, "");
+        }
+        else
+          break;
       }
     }
     ctx.expecting=old_expect;
@@ -1224,9 +1175,9 @@ std::pair<subshell*, parse_context> parse_subshell(parse_context ctx)
   {
 #endif
 ;
-    auto pp=parse_list_until(ctx, ')', ")");
-    ret->lst=pp.first;
-    ctx=pp.second;
+    auto pp=parse_list_until(ctx, {.end_char=')', .expecting=")"} );
+    ret->lst=std::get<0>(pp);
+    ctx=std::get<1>(pp);
     if(ret->lst->size()<=0)
     {
       parse_error("Subshell is empty", ctx, start-1);
@@ -1258,9 +1209,9 @@ std::pair<brace*, parse_context> parse_brace(parse_context ctx)
   {
 #endif
 ;
-    auto pp=parse_list_until(ctx, '}', "}");
-    ret->lst=pp.first;
-    ctx=pp.second;
+    auto pp=parse_list_until(ctx, {.end_char='}', .expecting="}"});
+    ret->lst=std::get<0>(pp);
+    ctx=std::get<1>(pp);
     if(ret->lst->size()<=0)
     {
       parse_error("Brace block is empty", ctx, start-1);
@@ -1299,16 +1250,16 @@ std::pair<function*, parse_context> parse_function(parse_context ctx, const char
     }
     ctx.i++;
 
-    auto pp=parse_list_until(ctx, '}', "}");
-    if(pp.first->size()<=0)
+    auto pp=parse_list_until(ctx, {.end_char='}', .expecting="}"} );
+    ret->lst=std::get<0>(pp);
+    if(ret->lst->size()<=0)
     {
       parse_error("Function is empty", ctx);
-      ctx.i=pp.second.i+1;
+      ctx.i=std::get<1>(pp).i+1;
       return std::make_pair(ret, ctx);
     }
 
-    ret->lst=pp.first;
-    ctx=pp.second;
+    ctx=std::get<1>(pp);
     ctx.i++;
 #ifndef NO_PARSE_CATCH
   }
@@ -1549,7 +1500,7 @@ std::pair<case_block*, parse_context> parse_case(parse_context ctx)
       ctx.i++;
 
       // until ;;
-      auto tp = parse_list_until(ctx, {";", "esac"}, ";;");
+      auto tp = parse_list_until(ctx, { .word_mode=true, .end_words={";", "esac"}, .expecting=";;" });
       cc->second = std::get<0>(tp);
       ctx = std::get<1>(tp);
       std::string word = std::get<2>(tp);
@@ -1600,20 +1551,21 @@ std::pair<if_block*, parse_context> parse_if(parse_context ctx)
     while(true)
     {
       std::string word;
+      parse_context oldctx = ctx;
 
       ret->blocks.push_back(std::make_pair(nullptr, nullptr));
       auto ll = ret->blocks.end()-1;
 
-      auto pp=parse_list_until(ctx, "then");
-      ll->first = pp.first;
+      auto pp=parse_list_until(ctx, {.word_mode=true, .end_words={"then"}});
+      ll->first = std::get<0>(pp);
+      ctx = std::get<1>(pp);
       if(ll->first->size()<=0)
       {
-        parse_error("Condition is empty", ctx);
-        pp.second.has_errored=true;
+        parse_error("Condition is empty", oldctx);
+        ctx.has_errored=true;
       }
-      ctx = pp.second;
 
-      auto tp=parse_list_until(ctx, {"fi", "elif", "else"});
+      auto tp=parse_list_until(ctx, {.word_mode=true, .end_words={"fi", "elif", "else"}} );
       ll->second = std::get<0>(tp);
       parse_context newctx = std::get<1>(tp);
       word = std::get<2>(tp);
@@ -1628,14 +1580,17 @@ std::pair<if_block*, parse_context> parse_if(parse_context ctx)
         break;
       if(word == "else")
       {
-        auto pp=parse_list_until(ctx, "fi");
-        if(pp.first->size()<=0)
+        auto pp=parse_list_until(ctx, {.word_mode=true, .end_words={"fi"}});
+        ret->else_lst=std::get<0>(pp);
+        if(ret->else_lst->size()<=0)
         {
           parse_error("else block is empty", ctx);
-          pp.second.has_errored=true;
+          ctx=std::get<1>(pp);
+          ctx.has_errored=true;
         }
-        ret->else_lst=pp.first;
-        ctx=pp.second;
+        else
+          ctx=std::get<1>(pp);
+
         break;
       }
 
@@ -1714,9 +1669,9 @@ std::pair<for_block*, parse_context> parse_for(parse_context ctx)
     }
 
     // ops
-    auto lp = parse_list_until(ctx, "done");
-    ret->ops=lp.first;
-    ctx=lp.second;
+    auto lp = parse_list_until(ctx, {.word_mode=true, .end_words={"done"}} );
+    ret->ops=std::get<0>(lp);
+    ctx=std::get<1>(lp);
 #ifndef NO_PARSE_CATCH
   }
   catch(format_error& e)
@@ -1739,24 +1694,28 @@ std::pair<while_block*, parse_context> parse_while(parse_context ctx)
 #endif
 ;
     // cond
-    auto pp=parse_list_until(ctx, "do");
-    if(pp.first->size() <= 0)
+    parse_context oldctx = ctx;
+    auto pp=parse_list_until(ctx, {.word_mode=true, .end_words={"do"}});
+
+    ret->cond = std::get<0>(pp);
+    ctx = std::get<1>(pp);
+
+    if(ret->cond->size() <= 0)
     {
-      parse_error("condition is empty", ctx);
-      pp.second.has_errored=true;
+      parse_error("condition is empty", oldctx);
+      ctx.has_errored=true;
     }
-    ret->cond = pp.first;
-    ctx = pp.second;
 
     // ops
-    auto lp = parse_list_until(ctx, "done");
-    if(lp.first->size() <= 0)
+    oldctx = ctx;
+    auto lp = parse_list_until(ctx, {.word_mode=true, .end_words={"done"}} );
+    ret->ops=std::get<0>(lp);
+    ctx = std::get<1>(lp);
+    if(ret->ops->size() <= 0)
     {
-      parse_error("while is empty", ctx);
-      lp.second.has_errored=true;
+      parse_error("while is empty", oldctx);
+      ctx.has_errored=true;
     }
-    ret->ops=lp.first;
-    ctx = lp.second;
 #ifndef NO_PARSE_CATCH
   }
   catch(format_error& e)
@@ -1936,13 +1895,14 @@ std::pair<shmain*, parse_context> parse_text(parse_context ctx)
   if(!ctx.bash)
     ctx.bash = (binshebang == "bash" || binshebang == "lxsh");
   // parse all commands
-  auto pp=parse_list_until(ctx, 0);
-  ret->lst=pp.first;
+  auto pp=parse_list_until(ctx);
+  ret->lst=std::get<0>(pp);
+  ctx = std::get<1>(pp);
 
-  if(pp.second.has_errored)
+  if(ctx.has_errored)
     throw std::runtime_error("Aborted due to previous errors");
 
-  return std::make_pair(ret, pp.second);
+  return std::make_pair(ret, ctx);
 }
 
 std::pair<shmain*, parse_context> parse_text(std::string const& in, std::string const& filename)
diff --git a/src/struc_helper.cpp b/src/struc_helper.cpp
index 335575b..cbeafdd 100644
--- a/src/struc_helper.cpp
+++ b/src/struc_helper.cpp
@@ -58,7 +58,8 @@ condlist* make_condlist(std::string const& in)
 
 list* make_list(std::string const& in)
 {
-  return parse_list_until(make_context(in), 0).first;
+  auto t = parse_list_until(make_context(in));
+  return std::get<0>(t);
 }
 
 block* make_block(std::string const& in)