feat(bash): support bash time syntax

- comment in pipeline throwing an error
- heredocument tries to parse quotes
- split pipeline/condlist combined with heredocuments don't parse correctly

.gitignore

@@ -1,10 +1,9 @@
 /obj
-/test
-/run-tests.sh
 /Zmakefile
-/TODO
+TODO
 /lxsh
-/gmon.out
+gmon.out
 /profiling/*
 /profiling.*
-/include/g_version.h
+/include/g_*
+/src/g_*

Makefile

@@ -14,58 +14,83 @@ LDFLAGS = -lpthread
 # compiler
 CC=g++
 # compiler flags
-CXXFLAGS= -I$(IDIR) -Wall -pedantic -std=c++17
+CXXFLAGS= -I$(IDIR) -Wall -std=c++20
 ifeq	($(DEBUG),true)
-  # debugging flags
-  CC=clang++
-  CXXFLAGS += -g -pg
+	# debugging flags
+	CXXFLAGS += -g -D DEBUG_MODE
+	RODIR = $(ODIR)/debug
 else
-  # release flags
-  CXXFLAGS += -Ofast
+	# release flags
+	CXXFLAGS += -Ofast
+	RODIR = $(ODIR)/release
+endif
+ifeq	($(STATIC),true)
+	# static links
+	LDFLAGS += -l:libztd.a
+else
+	# dynamic links
+	LDFLAGS += -lztd
+endif
+
+
+ifeq	($(PROFILE),true)
+  CXXFLAGS += -pg
 endif
 
 ifneq	($(RELEASE), true)
 	VSUFFIX=-dev-$(SHA_SHORT)
 endif
 
-ifeq    ($(STATIC),true)
-  # static links
-  LDFLAGS += -l:libztd.a
-else
-  # dynamic links
-  LDFLAGS += -lztd
-endif
-
 ## END CONFIG ##
 
+
 $(shell ./generate_version.sh)
-$(shell mkdir -p $(ODIR))
+$(shell ./generate_shellcode.sh)
+
+$(shell mkdir -p $(RODIR))
 $(shell mkdir -p $(BINDIR))
 
 # automatically find .h and .hpp
-DEPS = $(shell find $(IDIR) -type f -regex '.*\.hp?p?' ! -name 'g_version.h')
+DEPS = $(shell find $(IDIR) -type f -regex '.*\.hp?p?')
 # automatically find .c and .cpp and make the corresponding .o rule
-OBJ = $(shell find $(SRCDIR) -type f -regex '.*\.cp?p?' | sed 's|\.cpp|.o|g;s|\.c|.o|g;s|^$(SRCDIR)/|$(ODIR)/|g')
+OBJ = $(shell find $(SRCDIR) -type f -regex '.*\.cp?p?' | sed 's|\.cpp|.o|g;s|\.c|.o|g;s|^$(SRCDIR)/|$(RODIR)/|g')
 
-build: lxsh $(OBJ) $(DEPS)
+build: $(BINDIR)/$(NAME)
 
-$(ODIR)/%.o: $(SRCDIR)/%.c $(DEPS)
+# specific files for autogenerated headers
+$(OBJDIR)/options.o: $(SRCDIR)/options.cpp $(DEPS) $(IDIR)/g_version.h
 	$(CC) $(CXXFLAGS) -c -o $@ $<
 
-$(ODIR)/%.o: $(SRCDIR)/%.cpp $(DEPS)
+$(OBJDIR)/shellcode.o: $(SRCDIR)/shellcode.cpp $(DEPS) $(IDIR)/g_shellcode.h
 	$(CC) $(CXXFLAGS) -c -o $@ $<
 
-$(ODIR)/main.o: $(SRCDIR)/main.cpp $(DEPS) $(IDIR)/g_version.h
+$(OBJDIR)/debashify.o: $(SRCDIR)/debashify.cpp $(DEPS) $(IDIR)/g_shellcode.h
 	$(CC) $(CXXFLAGS) -c -o $@ $<
 
-lxsh: $(OBJ)
+# generic files
+
+$(RODIR)/%.o: $(SRCDIR)/%.c $(DEPS)
+	$(CC) $(CXXFLAGS) -c -o $@ $<
+
+$(RODIR)/%.o: $(SRCDIR)/%.cpp $(DEPS)
+	$(CC) $(CXXFLAGS) -c -o $@ $<
+
+
+$(BINDIR)/$(NAME): $(OBJ)
 	$(CC) $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
 
 test: $(BINDIR)/$(NAME)
 	$(BINDIR)/$(NAME)
 
 clean:
-	rm $(ODIR)/*.o
+	rm $(ODIR)/*/*.o
 
 clear:
 	rm $(BINDIR)/$(NAME)
+
+install:
+	cp lxsh /usr/local/bin
+	cp completion/lxsh.bash /etc/bash_completion.d
+
+uninstall:
+	rm /usr/local/bin/lxsh /etc/bash_completion.d/lxsh.bash

README.md

@@ -1,13 +1,29 @@
 # lxsh
 
-Extended shell linker for linking, processing and minimizing shell code
+Extended shell linker for linking, processing and minifying shell code
 
 # Installing
 
 ## linux-amd64
 
+### zpkg
+
+Available from the [zpkg](https://github.com/zawwz/zpkg) repository:
+```shell
+wget -qO- https://zpkg.zawz.net/install.sh | sh
+zpkg install lxsh
+```
+
+### Binary amd64
+
 Download the `lxsh-linux-amd64.tar.gz` archive, extract it,
-and move `lxsh` binary in a PATH folder (`/usr/local/bin` is the recommended).
+and move the `lxsh` binary in a PATH folder (`/usr/local/bin` is the recommended).
+
+```shell
+wget https://github.com/zawwz/lxsh/releases/download/v1.3.0/lxsh-linux-amd64.tar.gz
+tar -xvf lxsh-linux-amd64.tar.gz
+sudo mv lxsh /usr/local/bin
+```
 
 ## Other
 
@@ -15,9 +31,9 @@ See [Build](#build).
 
 # Features
 
-## Command extensions
+## Linking
 
-lxsh implements special linking commands that are resolved at linking.
+lxsh implements special linking commands that are resolved at build time.
 These commands can be placed anywhere within the script like regular commands.
 
 - `%include` : allows to insert file contents
@@ -25,23 +41,125 @@ These commands can be placed anywhere within the script like regular commands.
 
 > See `lxsh --help-commands` for more details
 
-## Minimize code
+## Minify code
 
 Reduce code size to a minimum without changing functionality with the `-m` option.
 
-### Further minimizing
+> This option should be safe to use in any situation, if this option changes behavior please report a bug
 
-The script can be further minimized by altering code elements.
+#### Behaviors of the minify option
+
+- removes any unnecessary separator character between arguments/commands
+- removes `;;` from the last value in a case
+- removes unnecessary quotes on arguments
+- transforms unnecessary manipulations (e.g. `${VAR}`) into simple variable call
+- brace blocks or subshells with a single command will be replaced by said command
+- reduces level 1 `$()` subshells to use backticks
+- escaped dollarsigns are un-escaped
+
+> These features only apply if they won't change behavior, for instance
+removal of an unnecessary manipulation will not be made if the following character
+could expand the variable name
+
+### Further minifying
+
+The script can be further minified by altering code elements.
 This can cause some change in execution behavior if you are not careful.
 
-Variable names can be minimized with `--minimize-var`,
-use `--exclude-var` to exclude variables from being minimized (for example environment config).
+Variable names can be minified with `--minify-var`,
+use `--exclude-var` to exclude variables from being minified (for example environment config).
 
-Function names can be minimized with `--minimize-fct`,
-use `--exclude-fct` to exclude functions from being minimized.
+Function names can be minified with `--minify-fct`,
+use `--exclude-fct` to exclude functions from being minified.
 
 Unused functions and variables can be removed with `--remove-unused`.
 
+Use `-M` to enable all of these minifying features (you still have to specify `--exclude` options when needed)
+
+## Debashify
+
+Some bash specific features can be translated into POSIX shell code.
+
+The following bash features can be debashified:
+- `<()` and `>()` process substitutions
+- `<<<` herestring
+- `>&`, `&>` and `&>>` output redirects
+- `[[ ]]` conditions
+- indexed arrays and associative arrays (+ their `declare` and `typeset` definitions)
+- `$RANDOM`
+- substring variable manipulation (`${VAR:N:M}`)
+- variable substitution (`${!VAR}`)
+- search replace manipulation (`${VAR/s/a/b}` )
+
+### Advantages
+
+- Removes dependency on bash and makes a script more portable.
+- In some cases it can also provide improved performance given that some more minimalist shells like `dash` have better performance.
+  * this doesn't always apply for all situations, make sure to verify through testing
+
+### Limitations
+
+#### $RANDOM
+
+Debashifying of $RANDOM assumes /dev/urandom exists and provides proper randomness. <br>
+The debashified $RANDOM generates numbers in range 0:65535 instead of 0:32767.
+
+Debashified calls of $RANDOM have major performance loss
+
+#### Process substitution
+
+The debashifying of process substitution assumes that /dev/urandom will exist and will provide proper randomness. <br>
+Temporary files with random names are used to create named pipes for the commands in the substitution.
+
+There may be some slight performance loss on the creation of said process subtitution.
+
+#### Indexed/Associative Arrays
+
+Indexed arrays and associative arrays are detected on parse instead of runtime.
+By default if an array operator is found, it is assumed to be an indexed array,
+and associative arrays are detected through the use of `declare` (or `typeset`). <br>
+In cases where there is ambiguity, the result upon execution might be undesired.
+
+> To avoid such ambiguities, put the `declare` statement of a variable first of all,
+> and don't mix and match different types on the same variable name
+
+Getting the value of an array without index will give the full value instead of the first value.
+
+> To avoid such situation, always get values from an index in your array
+
+Arrays are stored as strings. Indexed arrays are delimited by tabs and associative arrays by newlines,
+This means inserting values containing these characters will have undesired behavior.
+
+Debashified arrays have substantially reduced performance.
+
+Where bash would present proper errors upon incorrectly accessing arrays,
+these features will continue working with undesired behavior.
+
+> To avoid this, make sure to never access incorrect values
+
+Array argument with `[@]` does not expand into the desired multiple arguments.
+
+#### Substring manipulation
+
+Debashifying a substring manipulation on a variable containing a newline will not work correctly
+
+## Extension commands
+
+If you use the `#!/usr/bin/lxsh` shebang, you can use special lxsh-defined commands.
+To list such commands, see `lxsh --help-extend-fcts`
+
+## String processors
+
+You can use prefixes in singlequote strings to apply processing to the string contents. <br>
+To use string processors, prefix the string content with a line in the form of `#<PROCESSOR>`.
+Example:
+```shell
+sh -c '#LXSH_PARSE_MINIFY
+printf "%s\n" "Hello world!"'
+```
+
+As of now only the processor `LXSH_PARSE_MINIFY` is implemented, but more may come later
+
 ## Other features
 
 ### Output generated code
@@ -57,10 +175,8 @@ Directly execute an extended lxsh script with either
 - `-e` option
 - shebang is lxsh
 
-> Direct execution introduces direct dependency on lxsh and code generation overhead,
-> therefore it should be avoided outside of development use
-
-> There may be some issues with direct execution as of now
+> Direct execution introduces direct dependency on lxsh and code parsing overhead,
+> therefore it should be avoided in production environments.
 
 ### Variable/Function/command listing
 
@@ -74,7 +190,7 @@ Depends on [ztd](https://github.com/zawwz/ztd)
 
 ## Building
 
-Use `make -j8` to build.<br>
+Use `make -j` to build.<br>
 You can use environment variables to alter some aspects:
 - DEBUG: when set to `true` will generate a debug binary with profiling
 - RELEASE: when set to `true`, the version string will be generated for release format
@@ -83,23 +199,10 @@ You can use environment variables to alter some aspects:
 # Work in progress
 
 The full POSIX syntax is supported and should produce a functioning result. <br>
-However not all bash syntax is supported yet.
-
-## incomplete POSIX features
-
-- `$(())` arithmetics are not minimized
+Most bash syntax is also supported, but not all.
 
 ## Known bash issues
 
-- `&>` and `>&` are not supported
-- `|&` is not supported
-- `<<<` is not supported
-- `<()` is not supported
-- `${!VAR}` does not register as a variable
 - Extended globs (`*()`) are not supported
-- `(())` creates two subshells instead of one object
-- Unsetting functions will not work properly when minimizing variables or functions
-
-## Debashifying
-
-Work to debashify scripts is planned
+- `(())` is parsed as subshells
+- Unsetting functions can have undesired effects

completion/lxsh.bash

@@ -0,0 +1,3 @@
+#/usr/bin/env bash
+
+complete -F _longopt lxsh

generate_shellcode.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+file=include/g_shellcode.h
+tmpfile=${TMPDIR-/tmp}/lxsh_shellcodegen
+codedir=shellcode
+
+# $1 = file
+minify() {
+  if which lxsh >/dev/null 2>&1 ; then
+    lxsh -m "$1"
+  elif which shfmt >/dev/null 2>&1 ; then
+    shfmt -mn "$1"
+  else
+    cat "$1"
+  fi
+}
+
+to_cstr() {
+  sed 's|\\|\\\\|g;s|\"|\\\"|g' | sed ':a;N;$!ba;s/\n/\\n/g;'
+}
+
+
+cat > "$tmpfile" << EOF
+#ifndef G_VERSION_H
+#define G_VERSION_H
+EOF
+
+unset all_fields
+for I in "$codedir"/*.sh
+do
+  field=$(basename "$I" | tr [:lower:] [:upper:] | tr '.' '_')
+  all_fields="$all_fields $field"
+  printf '#define %s "%s\\n"\n' "$field" "$(minify "$I" | to_cstr)" >> "$tmpfile"
+done
+
+
+echo "#endif" >> "$tmpfile"
+
+if [ "$(md5sum "$tmpfile" | cut -d' ' -f1)" != "$(md5sum "$file" | cut -d' ' -f1)" ] ; then
+  mv "$tmpfile" "$file"
+else
+  rm "$tmpfile"
+fi

include/debashify.hpp

@@ -0,0 +1,22 @@
+#ifndef DEBASHIFY_HPP
+#define DEBASHIFY_HPP
+
+#include "struc.hpp"
+
+#include <map>
+#include <set>
+
+struct debashify_params {
+  std::set<std::string> required_fcts;
+  void require_fct(std::string const& in) { required_fcts.insert(in); }
+  // map of detected arrays
+  // bool value: is associative
+  std::map<std::string,bool> arrays;
+};
+
+bool r_debashify(_obj* o, debashify_params* params);
+
+std::set<std::string> debashify(_obj* o, debashify_params* params);
+std::set<std::string> debashify(shmain* sh);
+
+#endif //DEBASHIFY_HPP

include/errcodes.h

@@ -0,0 +1,9 @@
+#ifndef ERRCODES_H
+#define ERRCODES_H
+
+#define ERR_HELP    101
+#define ERR_OPT     102
+#define ERR_PARSE   103
+#define ERR_RUNTIME 104
+
+#endif //ERRCODES_H

include/exec.hpp

@@ -0,0 +1,12 @@
+#ifndef EXEC_HPP
+#define EXEC_HPP
+
+#include "options.hpp"
+#include "parse.hpp"
+
+
+void parse_exec(FILE* fd, parse_context ct);
+
+int exec_process(std::string const& runtime, std::vector<std::string> const& args, parse_context ct);
+
+#endif //EXEC_HPP

include/minify.hpp

@@ -0,0 +1,23 @@
+#ifndef MINIFY_HPP
+#define MINIFY_HPP
+
+#include "struc.hpp"
+#include "processing.hpp"
+
+#include <regex>
+#include <string>
+
+std::string gen_minmap(strmap_t const& map, std::string const& prefix);
+void read_minmap(std::string const& filepath, strmap_t* varmap, strmap_t* fctmap);
+
+bool r_replace_fct(_obj* in, strmap_t* fctmap);
+bool r_replace_var(_obj* in, strmap_t* varmap);
+
+strmap_t minify_var(_obj* in, std::regex const& exclude);
+strmap_t minify_fct(_obj* in, std::regex const& exclude);
+
+void delete_unused(_obj* in, std::regex const& var_exclude, std::regex const& fct_exclude);
+
+void minify_generic(_obj* in);
+
+#endif //MINIFY_HPP

include/minimize.hpp

@@ -1,16 +0,0 @@
-#ifndef MINIMIZE_HPP
-#define MINIMIZE_HPP
-
-#include "struc.hpp"
-
-#include <regex>
-#include <string>
-
-void minimize_var(_obj* in, std::regex const& exclude);
-void minimize_fct(_obj* in, std::regex const& exclude);
-
-bool delete_unused_fct(_obj* in, std::regex const& exclude);
-bool delete_unused_var(_obj* in, std::regex const& exclude);
-void delete_unused(_obj* in, std::regex const& var_exclude, std::regex const& fct_exclude);
-
-#endif //MINIMIZE_HPP

include/options.hpp

@@ -5,10 +5,13 @@
 
 extern ztd::option_set options;
 
-extern bool opt_minimize;
+extern bool opt_minify;
 extern bool g_cd;
 extern bool g_include;
 extern bool g_resolve;
+extern bool g_shebang;
+
+void print_lxsh_extension_help();
 
 void get_opts();
 
@@ -36,4 +39,6 @@ options:
 */
 ztd::option_set create_resolve_opts();
 
+void oneshot_opt_process(const char* arg0);
+
 #endif //OPTIONS_HPP

include/parse.hpp

@@ -4,27 +4,125 @@
 #include "struc.hpp"
 
 #include <string>
-
-#include <ztd/filedat.hpp>
+#include <utility>
+#include <vector>
+#include <set>
+#include <tuple>
 
 #define SPACES          " \t"
 #define SEPARATORS      " \t\n"
-#define ARG_END         " \t\n;#()&|<>"
+#define ARG_END         " \t\n;()&|<>"
 #define VARNAME_END     " \t\n;#()&|=\"'\\{}/-+"
 #define BLOCK_TOKEN_END " \t\n;#()&|=\"'\\"
+#define BASH_BLOCK_END  " \t\n;#()&|=\"'\\{}"
 #define COMMAND_SEPARATOR  "\n;"
 #define CONTROL_END           "#)"
 #define PIPELINE_END       "\n;#()&"
 #define ARGLIST_END        "\n;#()&|"
-#define SPECIAL_TOKENS     "\n;#()&|"
 #define ALL_TOKENS         "\n;#()&|{}"
 
+#define ARITHMETIC_OPERATOR_END " \t\n$)"
+
 #define SPECIAL_VARS "!#*@$?"
 
+// bash specific
+#define ARRAY_ARG_END       " \t\n;#()&|<>]"
+// optimizations
+#define ARG_OPTIMIZE_NULL     "$\\`"
+#define ARG_OPTIMIZE_MANIP    "$\\`}"
+#define ARG_OPTIMIZE_DEFARR   "$\\`)"
+#define ARG_OPTIMIZE_BASHTEST "$\\`] \t\n"
+#define ARG_OPTIMIZE_ARG      "$\\` \t\n;()&|<>\"'"
+#define ARG_OPTIMIZE_ARRAY    "$\\`\t\n&|}[]\"'"
+#define ARG_OPTIMIZE_ALL      "$\\` \t\n;#()&|<>}]\"'"
+
+// structs
+
+struct list_parse_options {
+  char end_char=0;
+  bool word_mode=false;
+  std::vector<std::string> end_words={};
+  const char* expecting=NULL;
+};
+
+// globals
+extern const std::set<std::string> all_reserved_words;
+extern const std::set<std::string> posix_cmdvar;
+extern const std::set<std::string> bash_cmdvar;
+
 std::string import_file(std::string const& path);
 
-shmain* parse_text(const char* in, uint32_t size, std::string const& filename="");
-inline shmain* parse_text(std::string const& in, std::string const& filename="") { return parse_text(in.c_str(), in.size(), filename); }
-inline shmain* parse(std::string const& file) { return parse_text(import_file(file), file); }
+std::pair<shmain*, parse_context> parse_text(parse_context context);
+std::pair<shmain*, parse_context> parse_text(std::string const& in, std::string const& filename="");
+inline std::pair<shmain*, parse_context> parse(std::string const& file) { return parse_text(import_file(file), file); }
+
+// tools
+
+parse_context make_context(std::string const& in, std::string const& filename="", bool bash=false);
+parse_context make_context(parse_context ctx, std::string const& in="", std::string const& filename="", bool bash=false);
+parse_context make_context(parse_context ctx, uint64_t i);
+parse_context operator+(parse_context ctx, int64_t a);
+parse_context operator-(parse_context ctx, int64_t a);
+
+// error handlers
+void parse_error(std::string const& message, parse_context& ctx);
+void parse_error(std::string const& message, parse_context& ctx, uint64_t i);
+
+// ** unit parsers ** //
+
+/* util parsers */
+uint32_t word_eq(const char* word, const char* in, uint32_t size, uint32_t start, const char* end_set=NULL);
+inline bool word_eq(const char* word, parse_context const& ct, const char* end_set=NULL) {
+  return word_eq(word, ct.data, ct.size, ct.i, end_set);
+}
+std::pair<std::string,uint32_t> get_word(parse_context ct, const char* end_set);
+uint32_t skip_chars(const char* in, uint32_t size, uint32_t start, const char* set);
+inline uint32_t skip_chars(parse_context const& ct, const char* set) {
+  return skip_chars(ct.data, ct.size, ct.i, set);
+}
+uint32_t skip_until(const char* in, uint32_t size, uint32_t start, const char* set);
+inline uint32_t skip_until(parse_context const& ct, const char* set) {
+  return skip_until(ct.data, ct.size, ct.i, set);
+}
+uint32_t skip_unread(const char* in, uint32_t size, uint32_t start);
+inline uint32_t skip_unread(parse_context const& ct) {
+  return skip_unread(ct.data, ct.size, ct.i);
+}
+uint32_t skip_unread_noline(const char* in, uint32_t size, uint32_t start);
+inline uint32_t skip_unread_noline(parse_context const& ct) {
+  return skip_unread_noline(ct.data, ct.size, ct.i);
+}
+
+// heredocument
+parse_context parse_heredocument(parse_context ctx);
+
+// list
+std::tuple<list_t*, parse_context, std::string> parse_list_until(parse_context ct, list_parse_options opts={});
+// name
+std::pair<variable_t*, parse_context> parse_var(parse_context ct, bool specialvars=true, bool array=false);
+
+// subarg parsers
+std::pair<arithmetic_t*, parse_context> parse_arithmetic(parse_context ct);
+std::pair<variable_t*, parse_context> parse_manipulation(parse_context ct);
+// arg parser
+std::pair<arg_t*, parse_context> parse_arg(parse_context ct, const char* end=ARG_END, const char* unexpected=ARGLIST_END, bool doquote=true, const char* optimize=ARG_OPTIMIZE_ARG);
+// redirect parser
+std::pair<redirect_t*, parse_context> parse_redirect(parse_context ct);
+// arglist parser
+std::pair<arglist_t*, parse_context> parse_arglist(parse_context ct, bool hard_error=false, std::vector<redirect_t*>* redirs=nullptr, bool stop_on_brace=false);
+// block parsers
+std::pair<block_t*, parse_context> parse_block(parse_context ct);
+std::pair<cmd_t*, parse_context> parse_cmd(parse_context ct);
+std::pair<function_t*, parse_context> parse_function(parse_context ct, const char* after="()");
+std::pair<subshell_t*, parse_context> parse_subshell(parse_context ct);
+std::pair<brace_t*, parse_context> parse_brace(parse_context ct);
+std::pair<case_t*, parse_context> parse_case(parse_context ct);
+std::pair<if_t*, parse_context> parse_if(parse_context ct);
+std::pair<for_t*, parse_context> parse_for(parse_context ct);
+std::pair<while_t*, parse_context> parse_while(parse_context ct);
+// pipeline parser
+std::pair<pipeline_t*, parse_context> parse_pipeline(parse_context ct);
+// condlist parser
+std::pair<condlist_t*, parse_context> parse_condlist(parse_context ct);
 
 #endif //PARSE_HPP

include/processing.hpp

@@ -9,7 +9,7 @@
 #include "struc.hpp"
 
 // constants
-#define RESERVED_VARIABLES "HOME", "PATH", "SHELL", "PWD", "OPTIND", "OPTARG"
+#define RESERVED_VARIABLES "HOME", "PATH", "SHELL", "PWD", "OPTIND", "OPTARG", "LC_.*", "LANG", "TERM", "RANDOM", "TMPDIR", "IFS"
 
 // types
 typedef std::map<std::string,uint32_t> countmap_t;
@@ -26,8 +26,13 @@ extern countmap_t m_vars, m_vardefs, m_varcalls;
 extern countmap_t m_fcts, m_cmds;
 extern set_t m_excluded_var, m_excluded_fct, m_excluded_cmd;
 
+
 extern bool b_gotvar, b_gotfct, b_gotcmd;
 
+// tools
+countmap_t combine_maps(countmap_t const& a, countmap_t const& b);
+countmap_t combine_common(countmap_t const& a, countmap_t const& b);
+
 /** map get functions (optimizations) **/
 
 // rescans
@@ -40,8 +45,13 @@ void require_rescan_cmd();
 void varmap_get(_obj* in, std::regex const& exclude);
 void fctmap_get(_obj* in, std::regex const& exclude);
 void cmdmap_get(_obj* in, std::regex const& exclude);
+void fctcmdmap_get(_obj* in, std::regex const& exclude_fct, std::regex const& exclude_cmd);
+void allmaps_get(_obj* in, std::regex const& exclude_var, std::regex const& exclude_fct, std::regex const& exclude_cmd);
 
 /** util functions **/
+#ifdef DEBUG_MODE
+std::string gen_json_struc(_obj* in);
+#endif
 
 // gen regexes
 std::regex var_exclude_regex(std::string const& in, bool include_reserved);
@@ -50,7 +60,7 @@ std::regex fct_exclude_regex(std::string const& in);
 // varnames
 bool is_varname(std::string const& in);
 std::string get_varname(std::string const& in);
-std::string get_varname(arg* in);
+std::string get_varname(arg_t* in);
 
 // list objects
 void list_map(countmap_t const& map);
@@ -61,15 +71,24 @@ void list_fcts(_obj* in, std::regex const& exclude);
 void list_cmds(_obj* in, std::regex const& exclude);
 
 // recursives
+bool r_has_env_set(_obj* in, bool* result);
 bool r_get_unsets(_obj* in, set_t* unsets);
 bool r_get_var(_obj* in, countmap_t* defmap, countmap_t* callmap);
 bool r_get_cmd(_obj* in, countmap_t* all_cmds);
 bool r_get_fct(_obj* in, countmap_t* fct_map);
+bool r_get_fctcmd(_obj* in, countmap_t* all_cmds, countmap_t* fct_map);
+bool r_get_all(_obj* in, countmap_t* defmap, countmap_t* callmap, countmap_t* all_cmds, countmap_t* fct_map);
 bool r_delete_fct(_obj* in, set_t* fcts);
 bool r_delete_var(_obj* in, set_t* vars);
+bool r_delete_varfct(_obj* in, set_t* vars, set_t* fcts);
+bool r_do_string_processor(_obj* in);
 
 /** Processing **/
 
+std::set<std::string> find_lxsh_commands(shmain* sh);
 void add_unset_variables(shmain* sh, std::regex const& exclude);
+bool has_env_set(_obj* in);
+
+void string_processors(_obj* in);
 
 #endif //PROCESSING_HPP

include/recursive.hpp

@@ -19,60 +19,58 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
   // recursive calls
   switch(o->type)
   {
-    case _obj::_redirect :
+    case _obj::variable :
     {
-      redirect* t = dynamic_cast<redirect*>(o);
+      variable_t* t = dynamic_cast<variable_t*>(o);
+      recurse(fct, t->index, args...);
+      recurse(fct, t->manip, args...);
+      break;
+    }
+    case _obj::redirect :
+    {
+      redirect_t* t = dynamic_cast<redirect_t*>(o);
       recurse(fct, t->target, args...);
+      recurse(fct, t->here_document, args...);
       break;
     }
-    case _obj::_arg :
+    case _obj::arg :
     {
-      arg* t = dynamic_cast<arg*>(o);
+      arg_t* t = dynamic_cast<arg_t*>(o);
       for(auto it: t->sa)
-      {
         recurse(fct, it, args...);
-      }
       break;
     }
-    case _obj::_arglist :
+    case _obj::arglist :
     {
-      arglist* t = dynamic_cast<arglist*>(o);
+      arglist_t* t = dynamic_cast<arglist_t*>(o);
       for(auto it: t->args)
-      {
         recurse(fct, it, args...);
-      }
       break;
     }
-    case _obj::_pipeline :
+    case _obj::pipeline :
     {
-      pipeline* t = dynamic_cast<pipeline*>(o);
+      pipeline_t* t = dynamic_cast<pipeline_t*>(o);
       for(auto it: t->cmds)
-      {
         recurse(fct, it, args...);
-      }
       break;
     }
-    case _obj::_condlist :
+    case _obj::condlist :
     {
-      condlist* t = dynamic_cast<condlist*>(o);
+      condlist_t* t = dynamic_cast<condlist_t*>(o);
       for(auto it: t->pls)
-      {
         recurse(fct, it, args...);
-      }
       break;
     }
-    case _obj::_list :
+    case _obj::list :
     {
-      list* t = dynamic_cast<list*>(o);
+      list_t* t = dynamic_cast<list_t*>(o);
       for(auto it: t->cls)
-      {
         recurse(fct, it, args...);
-      }
       break;
     }
     case _obj::block_subshell :
     {
-      subshell* t = dynamic_cast<subshell*>(o);
+      subshell_t* t = dynamic_cast<subshell_t*>(o);
       recurse(fct, t->lst, args...);
 
       for(auto it: t->redirs)
@@ -82,7 +80,7 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
     }
     case _obj::block_brace :
     {
-      brace* t = dynamic_cast<brace*>(o);
+      brace_t* t = dynamic_cast<brace_t*>(o);
       recurse(fct, t->lst, args...);
 
       for(auto it: t->redirs)
@@ -102,7 +100,7 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
     }
     case _obj::block_function :
     {
-      function* t = dynamic_cast<function*>(o);
+      function_t* t = dynamic_cast<function_t*>(o);
       recurse(fct, t->lst, args...);
 
       for(auto it: t->redirs)
@@ -112,10 +110,18 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
     }
     case _obj::block_cmd :
     {
-      cmd* t = dynamic_cast<cmd*>(o);
+      cmd_t* t = dynamic_cast<cmd_t*>(o);
       recurse(fct, t->args, args...);
       for(auto it: t->var_assigns)
+      {
+        recurse(fct, it.first, args...);
         recurse(fct, it.second, args...);
+      }
+      for(auto it: t->cmd_var_assigns)
+      {
+        recurse(fct, it.first, args...);
+        recurse(fct, it.second, args...);
+      }
 
       for(auto it: t->redirs)
         recurse(fct, it, args...);
@@ -124,7 +130,7 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
     }
     case _obj::block_case :
     {
-      case_block* t = dynamic_cast<case_block*>(o);
+      case_t* t = dynamic_cast<case_t*>(o);
       // carg
       recurse(fct, t->carg, args...);
       // cases
@@ -144,7 +150,7 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
     }
     case _obj::block_if :
     {
-      if_block* t = dynamic_cast<if_block*>(o);
+      if_t* t = dynamic_cast<if_t*>(o);
       // ifs
       for(auto sc: t->blocks)
       {
@@ -163,7 +169,9 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
     }
     case _obj::block_for :
     {
-      for_block* t = dynamic_cast<for_block*>(o);
+      for_t* t = dynamic_cast<for_t*>(o);
+      // variable
+      recurse(fct, t->var, args...);
       // iterations
       recurse(fct, t->iter, args...);
       // for block
@@ -176,7 +184,7 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
     }
     case _obj::block_while :
     {
-      while_block* t = dynamic_cast<while_block*>(o);
+      while_t* t = dynamic_cast<while_t*>(o);
       // condition
       recurse(fct, t->cond, args...);
       // operations
@@ -187,16 +195,346 @@ void recurse(bool (&fct)(_obj*, Args...), _obj* o, Args... args)
 
       break;
     }
+    case _obj::subarg_variable :
+    {
+      subarg_variable_t* t = dynamic_cast<subarg_variable_t*>(o);
+      recurse(fct, t->var, args...);
+      break;
+    }
     case _obj::subarg_subshell :
     {
-      subshell_subarg* t = dynamic_cast<subshell_subarg*>(o);
+      subarg_subshell_t* t = dynamic_cast<subarg_subshell_t*>(o);
       recurse(fct, t->sbsh, args...);
       break;
     }
+    case _obj::subarg_procsub :
+    {
+      subarg_procsub_t* t = dynamic_cast<subarg_procsub_t*>(o);
+      recurse(fct, t->sbsh, args...);
+      break;
+    }
+    case _obj::subarg_arithmetic :
+    {
+      subarg_arithmetic_t* t = dynamic_cast<subarg_arithmetic_t*>(o);
+      recurse(fct, t->arith, args...);
+      break;
+    }
+    case _obj::arithmetic_variable :
+    {
+      arithmetic_variable_t* t = dynamic_cast<arithmetic_variable_t*>(o);
+      recurse(fct, t->var, args...);
+      break;
+    }
+    case _obj::arithmetic_subshell :
+    {
+      arithmetic_subshell_t* t = dynamic_cast<arithmetic_subshell_t*>(o);
+      recurse(fct, t->sbsh, args...);
+      break;
+    }
+    case _obj::arithmetic_operation :
+    {
+      arithmetic_operation_t* t = dynamic_cast<arithmetic_operation_t*>(o);
+      recurse(fct, t->val1, args...);
+      recurse(fct, t->val2, args...);
+      break;
+    }
+    case _obj::arithmetic_parenthesis :
+    {
+      arithmetic_parenthesis_t* t = dynamic_cast<arithmetic_parenthesis_t*>(o);
+      recurse(fct, t->val, args...);
+      break;
+    }
 
     default: break; //do nothing
   }
 }
 
+// deep copy of object structure
+template<class... Args>
+_obj* obj_copy(_obj* o)
+{
+  if(o == nullptr)
+    return nullptr;
+
+  // recursive calls
+  switch(o->type)
+  {
+    case _obj::variable :
+    {
+      variable_t* t = dynamic_cast<variable_t*>(o);
+      variable_t* ret = new variable_t(*t);
+
+      ret->index = dynamic_cast<arg_t*>(obj_copy(t->index));
+      ret->manip = dynamic_cast<arg_t*>(obj_copy(t->manip));
+      return ret;
+    }
+    case _obj::redirect :
+    {
+      redirect_t* t = dynamic_cast<redirect_t*>(o);
+      redirect_t* ret = new redirect_t(*t);
+
+      ret->target = dynamic_cast<arg_t*>(obj_copy(t->target));
+      ret->here_document = dynamic_cast<arg_t*>(obj_copy(t->here_document));
+      return ret;
+    }
+    case _obj::arg :
+    {
+      arg_t* t = dynamic_cast<arg_t*>(o);
+      arg_t* ret = new arg_t(*t);
+
+      for(uint32_t i=0; i<ret->sa.size(); i++)
+        ret->sa[i] = dynamic_cast<subarg_t*>(obj_copy(t->sa[i]));
+      return ret;
+    }
+    case _obj::arglist :
+    {
+      arglist_t* t = dynamic_cast<arglist_t*>(o);
+      arglist_t* ret = new arglist_t(*t);
+
+      for(uint32_t i=0; i<ret->args.size(); i++)
+        ret->args[i] = dynamic_cast<arg_t*>(obj_copy(t->args[i]));
+      return ret;
+    }
+    case _obj::pipeline :
+    {
+      pipeline_t* t = dynamic_cast<pipeline_t*>(o);
+      pipeline_t* ret = new pipeline_t(*t);
+
+      for(uint32_t i=0; i<ret->cmds.size(); i++)
+        ret->cmds[i] = dynamic_cast<block_t*>(obj_copy(t->cmds[i]));
+      return ret;
+    }
+    case _obj::condlist :
+    {
+      condlist_t* t = dynamic_cast<condlist_t*>(o);
+      condlist_t* ret = new condlist_t(*t);
+
+      for(uint32_t i=0; i<ret->pls.size(); i++)
+        ret->pls[i] = dynamic_cast<pipeline_t*>(obj_copy(t->pls[i]));
+      return ret;
+    }
+    case _obj::list :
+    {
+      list_t* t = dynamic_cast<list_t*>(o);
+      list_t* ret = new list_t(*t);
+
+      for(uint32_t i=0; i<ret->cls.size(); i++)
+        ret->cls[i] = dynamic_cast<condlist_t*>(obj_copy(t->cls[i]));
+      return ret;
+    }
+    case _obj::block_subshell :
+    {
+      subshell_t* t = dynamic_cast<subshell_t*>(o);
+      subshell_t* ret = new subshell_t(*t);
+
+      ret->lst = dynamic_cast<list_t*>(obj_copy(t->lst));
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::block_brace :
+    {
+      brace_t* t = dynamic_cast<brace_t*>(o);
+      brace_t* ret = new brace_t(*t);
+
+      ret->lst = dynamic_cast<list_t*>(obj_copy(t->lst));
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::block_main :
+    {
+      shmain* t = dynamic_cast<shmain*>(o);
+      shmain* ret = new shmain(*t);
+
+      ret->lst = dynamic_cast<list_t*>(obj_copy(t->lst));
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::block_function :
+    {
+      function_t* t = dynamic_cast<function_t*>(o);
+      function_t* ret = new function_t(*t);
+
+      ret->lst = dynamic_cast<list_t*>(obj_copy(t->lst));
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::block_cmd :
+    {
+      cmd_t* t = dynamic_cast<cmd_t*>(o);
+      cmd_t* ret = new cmd_t(*t);
+
+      ret->args = dynamic_cast<arglist_t*>(obj_copy(t->args));
+
+      for(uint32_t i=0; i<ret->var_assigns.size(); i++)
+      {
+        ret->var_assigns[i].first = dynamic_cast<variable_t*>(obj_copy(t->var_assigns[i].first));
+        ret->var_assigns[i].second = dynamic_cast<arg_t*>(obj_copy(t->var_assigns[i].second));
+      }
+      for(uint32_t i=0; i<ret->cmd_var_assigns.size(); i++)
+      {
+        ret->cmd_var_assigns[i].first = dynamic_cast<variable_t*>(obj_copy(t->cmd_var_assigns[i].first));
+        ret->cmd_var_assigns[i].second = dynamic_cast<arg_t*>(obj_copy(t->cmd_var_assigns[i].second));
+      }
+
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::block_case :
+    {
+      case_t* t = dynamic_cast<case_t*>(o);
+      case_t* ret = new case_t(*t);
+
+      // carg
+      ret->carg = dynamic_cast<arg_t*>(obj_copy(t->carg));
+      // cases
+      for(uint32_t i=0; i<ret->cases.size(); i++)
+      {
+        for(uint32_t j=0; j<ret->cases[i].first.size(); i++)
+          ret->cases[i].first[j] = dynamic_cast<arg_t*>(obj_copy(t->cases[i].first[j]));
+        ret->cases[i].second = dynamic_cast<list_t*>(obj_copy(t->cases[i].second));
+      }
+
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::block_if :
+    {
+      if_t* t = dynamic_cast<if_t*>(o);
+      if_t* ret = new if_t(*t);
+
+      // ifs
+      for(uint32_t i=0; i<ret->blocks.size(); i++)
+      {
+        // condition
+        ret->blocks[i].first = dynamic_cast<list_t*>(obj_copy(t->blocks[i].first));
+        // execution
+        ret->blocks[i].second = dynamic_cast<list_t*>(obj_copy(t->blocks[i].second));
+      }
+      // else
+      ret->else_lst = dynamic_cast<list_t*>(obj_copy(t->else_lst));
+
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::block_for :
+    {
+      for_t* t = dynamic_cast<for_t*>(o);
+      for_t* ret = new for_t(*t);
+
+      // variable
+      ret->var = dynamic_cast<variable_t*>(obj_copy(t->var));
+      // iterations
+      ret->iter = dynamic_cast<arglist_t*>(obj_copy(t->iter));
+      // for block
+      ret->ops = dynamic_cast<list_t*>(obj_copy(t->ops));
+
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::block_while :
+    {
+      while_t* t = dynamic_cast<while_t*>(o);
+      while_t* ret = new while_t(*t);
+
+      // condition
+      ret->cond = dynamic_cast<list_t*>(obj_copy(t->cond));
+      // for operations
+      ret->ops = dynamic_cast<list_t*>(obj_copy(t->ops));
+
+      for(uint32_t i=0; i<ret->redirs.size(); i++)
+        ret->redirs[i] = dynamic_cast<redirect_t*>(obj_copy(t->redirs[i]));
+
+      return ret;
+    }
+    case _obj::subarg_string :
+    {
+      subarg_string_t* t = dynamic_cast<subarg_string_t*>(o);
+      subarg_string_t* ret = new subarg_string_t(*t);
+      return ret;
+    }
+    case _obj::subarg_variable :
+    {
+      subarg_variable_t* t = dynamic_cast<subarg_variable_t*>(o);
+      subarg_variable_t* ret = new subarg_variable_t(*t);
+      ret->var = dynamic_cast<variable_t*>(obj_copy(t->var));
+      return ret;
+    }
+    case _obj::subarg_subshell :
+    {
+      subarg_subshell_t* t = dynamic_cast<subarg_subshell_t*>(o);
+      subarg_subshell_t* ret = new subarg_subshell_t(*t);
+      ret->sbsh = dynamic_cast<subshell_t*>(obj_copy(t->sbsh));
+      return ret;
+    }
+    case _obj::subarg_procsub :
+    {
+      subarg_procsub_t* t = dynamic_cast<subarg_procsub_t*>(o);
+      subarg_procsub_t* ret = new subarg_procsub_t(*t);
+      ret->sbsh = dynamic_cast<subshell_t*>(obj_copy(t->sbsh));
+      return ret;
+    }
+    case _obj::subarg_arithmetic :
+    {
+      subarg_arithmetic_t* t = dynamic_cast<subarg_arithmetic_t*>(o);
+      subarg_arithmetic_t* ret = new subarg_arithmetic_t(*t);
+      ret->arith = dynamic_cast<arithmetic_t*>(obj_copy(t->arith));
+      return ret;
+    }
+    case _obj::arithmetic_number :
+    {
+      arithmetic_number_t* t = dynamic_cast<arithmetic_number_t*>(o);
+      arithmetic_number_t* ret = new arithmetic_number_t(*t);
+      return ret;
+    }
+    case _obj::arithmetic_variable :
+    {
+      arithmetic_variable_t* t = dynamic_cast<arithmetic_variable_t*>(o);
+      arithmetic_variable_t* ret = new arithmetic_variable_t(*t);
+      ret->var = dynamic_cast<variable_t*>(obj_copy(t->var));
+      return ret;
+    }
+    case _obj::arithmetic_subshell :
+    {
+      arithmetic_subshell_t* t = dynamic_cast<arithmetic_subshell_t*>(o);
+      arithmetic_subshell_t* ret = new arithmetic_subshell_t(*t);
+      ret->sbsh = dynamic_cast<subshell_t*>(obj_copy(t->sbsh));
+      return ret;
+    }
+    case _obj::arithmetic_operation :
+    {
+      arithmetic_operation_t* t = dynamic_cast<arithmetic_operation_t*>(o);
+      arithmetic_operation_t* ret = new arithmetic_operation_t(*t);
+      ret->val1 = dynamic_cast<arithmetic_t*>(obj_copy(t->val1));
+      ret->val2 = dynamic_cast<arithmetic_t*>(obj_copy(t->val2));
+      return ret;
+    }
+    case _obj::arithmetic_parenthesis :
+    {
+      arithmetic_parenthesis_t* t = dynamic_cast<arithmetic_parenthesis_t*>(o);
+      arithmetic_parenthesis_t* ret = new arithmetic_parenthesis_t(*t);
+      ret->val = dynamic_cast<arithmetic_t*>(obj_copy(t->val));
+      return ret;
+    }
+
+    default: return nullptr; //dummy
+  }
+}
+
 
 #endif //RECURSIVE_HPP

include/resolve.hpp

@@ -2,12 +2,18 @@
 #define RESOLVE_HPP
 
 #include "struc.hpp"
+#include "parse.hpp"
 
 extern std::vector<std::string> included;
 
+std::vector<std::pair<std::string, std::string>> do_include_raw(condlist_t* cmd, parse_context ctx, std::string* ex_dir=nullptr);
+std::pair<std::string, std::string> do_resolve_raw(condlist_t* cmd, parse_context ctx, std::string* ex_dir=nullptr);
+
 bool add_include(std::string const& file);
 
-void resolve(_obj* sh, shmain* parent);
-void resolve(shmain* sh);
+void resolve(_obj* sh, parse_context ctx);
+
+std::string _pre_cd(std::string const& filename);
+void _cd(std::string const& dir);
 
 #endif //RESOLVE_HPP

include/shellcode.hpp

@@ -0,0 +1,24 @@
+#ifndef SHELLCODE_HPP
+#define SHELLCODE_HPP
+
+#include <string>
+#include <set>
+#include <vector>
+#include <map>
+
+#include "struc.hpp"
+
+struct lxsh_fct {
+  std::string arguments;
+  std::string description;
+  const char* code;
+  std::vector<std::string> depends_on=std::vector<std::string>();
+};
+
+extern const std::map<const std::string, const lxsh_fct> lxsh_extend_fcts;
+extern const std::map<const std::string, const lxsh_fct> lxsh_array_fcts;
+extern const std::map<const std::string, const lxsh_fct> lxsh_allfcts;
+
+void add_lxsh_fcts(shmain* sh, std::set<std::string> fcts);
+
+#endif //SHELLCODE_HPP

include/struc.hpp

@@ -54,298 +54,425 @@ arg:
 
 subarg: can be one of
 - string
-- block: subshell (substitution)
+- subshell (command substitution)
 - arithmetic
+- variable
+- procsub (bash specific process substitution)
+  > NOTE: MUST be the only subarg in the arg
 
 */
 
+// pre-definitions
+
 #define AND_OP false
 #define OR_OP  true
 
-class condlist;
-class block;
-class pipeline;
-class arg;
-class subarg;
-class cmd;
+class condlist_t;
+class block_t;
+class pipeline_t;
+class arg_t;
+class subarg_t;
+class cmd_t;
+class redirect_t;
 
-// type pack of condlist
-typedef std::vector<arg*> arglist_t;
+// structs
 
-extern std::string g_origin;
+struct parse_context {
+  const char* data=NULL;
+  uint64_t size=0;
+  uint64_t i=0;
+  const char* filename="";
+  bool bash=false;
+  const char* expecting="";
+  const char* here_delimiter="";
+  const char* here_doc="";
+  const char operator[](uint64_t a) { return data[a]; }
+  bool has_errored=false;
+  redirect_t* here_document=nullptr;
+  char* here_delimitor=NULL;
+};
 
-cmd* make_cmd(std::vector<std::string> args);
+struct generate_context {
+  arg_t* here_document=nullptr;
+};
+
+// exceptions
+
+class format_error : public std::exception
+{
+public:
+  //! @brief Conctructor
+  inline format_error(const std::string& what, const std::string& origin, const std::string& data, int where, std::string level="error")  { desc=what; index=where; filename=origin; sdat=data; severity=level; }
+  inline format_error(const std::string& what, parse_context const& ctx, std::string level="error") { desc=what; index=ctx.i; filename=ctx.filename; sdat=ctx.data; severity=level; }
+  //! @brief Error message
+  inline const char * what () const throw () {return desc.c_str();}
+  //! @brief Origin of the data, name of imported file, otherwise empty if generated
+  inline const char * origin() const throw () {return filename.c_str();}
+  //! @brief Data causing the exception
+  inline const char * data() const throw () {return sdat.c_str();}
+  //! @brief Severity of the exception
+  inline const std::string level() const throw () {return severity.c_str();}
+  //! @brief Where the error is located in the data
+  inline const int where () const throw () {return index;}
+private:
+  std::string desc;
+  int index;
+  std::string filename;
+  std::string sdat;
+  std::string severity;
+};
+
+// objects
 
 // meta object type
 class _obj
 {
 public:
   enum _objtype {
-    subarg_string, subarg_variable, subarg_subshell, subarg_arithmetic, subarg_manipulation,
-    _redirect,
-    _arg,
-    _arglist,
-    _pipeline,
-    _condlist,
-    _list,
-    block_subshell, block_brace, block_main, block_cmd, block_function, block_case, block_if, block_for, block_while, block_until };
+    subarg_string, subarg_variable, subarg_subshell, subarg_arithmetic, subarg_procsub,
+    variable,
+    redirect,
+    arg,
+    arglist,
+    pipeline,
+    condlist,
+    list,
+    arithmetic_operation, arithmetic_number, arithmetic_variable, arithmetic_parenthesis, arithmetic_subshell,
+    block_subshell, block_brace, block_main, block_cmd, block_function, block_case, block_if, block_for, block_while };
   _objtype type;
 
   virtual ~_obj() {;}
   virtual std::string generate(int ind)=0;
 };
 
-// meta subarg type
-class subarg : public _obj
+// meta arithmetic type
+class arithmetic_t : public _obj
 {
 public:
-  virtual ~subarg() {;}
   virtual std::string generate(int ind)=0;
 };
 
-class arg : public _obj
+// meta subarg type
+class subarg_t : public _obj
 {
 public:
-  arg() { type=_obj::_arg; }
-  arg(std::string const& str) { type=_obj::_arg; this->setstring(str);}
-  ~arg() { for(auto it: sa) delete it; }
+  virtual ~subarg_t() {;}
+  virtual std::string generate(int ind)=0;
 
-  void setstring(std::string const& str);
+  bool quoted;
+};
 
-  std::vector<subarg*> sa;
+class arg_t : public _obj
+{
+public:
+  arg_t() { type=_obj::arg; forcequoted=false; }
+  arg_t(std::string const& str, bool fquote=false) { type=_obj::arg; this->set(str); forcequoted=fquote; }
+  arg_t(subarg_t* in, bool fquote=false) { type=_obj::arg; sa.push_back(in); forcequoted=fquote; }
+  ~arg_t() { for(auto it: sa) delete it; }
 
+  void set(std::string const& str);
+
+  void insert(uint32_t i, subarg_t* val);
+  void insert(uint32_t i, arg_t const& a);
+  void insert(uint32_t i, std::string const& in);
+
+  inline void add(subarg_t* in) { sa.push_back(in); }
+  void add(std::string const& in);
+  inline size_t size() { return sa.size(); }
+
+  std::vector<subarg_t*> sa;
+
+  // is forcequoted: var assign
+  bool forcequoted;
+
+  bool is_string();
   // return if is a string and only one subarg
   std::string string();
+  // return if the first subarg is a string
+  std::string first_sa_string();
+
+  // can expand into multiple arguments
+  bool can_expand();
 
   inline bool equals(std::string const& in) { return this->string() == in; }
 
   std::string generate(int ind);
 };
 
-inline bool operator==(arg a, std::string const& b) { return a.equals(b); }
-
-// arglist
-
-class arglist : public _obj
+class variable_t : public _obj
 {
 public:
-  arglist() { type=_obj::_arglist; }
-  arglist(arg* in) { type=_obj::_arglist; this->add(in); }
-  ~arglist() { for( auto it: args ) delete it; }
-  inline void add(arg* in) { args.push_back(in); }
-  inline void push_back(arg* in) { args.push_back(in); }
+  variable_t(std::string const& in="", arg_t* i=nullptr, bool def=false, bool ismanip=false, arg_t* m=nullptr) { type=_obj::variable; varname=in; index=i; definition=def; is_manip=ismanip; precedence=false; manip=m; }
+  ~variable_t() {
+    if(index!=nullptr) delete index;
+    if(manip!=nullptr) delete manip;
+  }
 
-  std::vector<arg*> args;
+  std::string varname;
+  bool definition;
+  arg_t* index; // for bash specific
 
-  std::vector<std::string> strargs(uint32_t start);
-
-  inline uint64_t size() { return args.size(); }
-  inline arg* operator[](uint32_t i) { return args[i]; }
+  bool is_manip;
+  bool precedence;
+  arg_t* manip;
 
   std::string generate(int ind);
 };
 
-class redirect : public _obj
+// arglist
+
+class arglist_t : public _obj
 {
 public:
-  redirect(arg* in=nullptr) { type=_obj::_redirect; target=in; }
-  ~redirect() { if(target != nullptr) delete target; }
+  arglist_t() { type=_obj::arglist; }
+  arglist_t(arg_t* in) { type=_obj::arglist; this->add(in); }
+  ~arglist_t() { for( auto it: args ) delete it; }
+  inline void add(arg_t* in) { args.push_back(in); }
+
+  std::vector<arg_t*> args;
+
+  std::vector<std::string> strargs(uint32_t start);
+
+  // get first argument as string
+  std::string first_arg_string();
+  // potentially expands into more arguments than its size
+  bool can_expand();
+
+  void insert(uint32_t i, arg_t* val);
+  void insert(uint32_t i, arglist_t const& lst);
+
+  inline size_t size() { return args.size(); }
+
+  std::string generate(int ind);
+};
+
+class redirect_t : public _obj
+{
+public:
+  redirect_t(std::string strop="") { type=_obj::redirect; op=strop; target=nullptr; here_document=nullptr; }
+  redirect_t(arg_t* in) { type=_obj::redirect; target=in; here_document=nullptr; }
+  redirect_t(std::string strop, arg_t* in) { type=_obj::redirect; op=strop; target=in; here_document=nullptr; }
+  redirect_t(std::string strop, arg_t* in, arg_t* doc) { type=_obj::redirect; op=strop; target=in; here_document=doc; }
+  ~redirect_t() {
+    if(target != nullptr) delete target;
+    if(here_document != nullptr) delete here_document;
+  }
 
   std::string generate(int ind);
 
   std::string op;
-  arg* target;
+  arg_t* target;
+  arg_t* here_document;
 };
 
 // Meta block
-class block : public _obj
+class block_t : public _obj
 {
 public:
-  block() { ; }
-  virtual ~block() { for(auto it: redirs) delete it; }
+  block_t() { ; }
+  virtual ~block_t() { for(auto it: redirs) delete it; }
   // cmd
-  std::vector<redirect*> redirs;
+  std::vector<redirect_t*> redirs;
 
   // subshell: return the containing cmd, if it is a single command
-  cmd* single_cmd();
+  cmd_t* single_cmd();
 
-  std::string generate_redirs(int ind, std::string const& _str);
+  std::string generate_redirs(int ind, std::string const& _str, generate_context* ctx);
 
-  virtual std::string generate(int ind)=0;
+  virtual std::string generate(int ind, generate_context* ctx)=0;
 };
 
 // PL
 
-class pipeline : public _obj
+class pipeline_t : public _obj
 {
 public:
-  pipeline(block* bl=nullptr) { type=_obj::_pipeline; if(bl!=nullptr) cmds.push_back(bl); negated=false; }
-  ~pipeline() { for(auto it: cmds) delete it; }
-  inline void add(block* bl) { this->cmds.push_back(bl); }
-  std::vector<block*> cmds;
+  pipeline_t(block_t* bl=nullptr) { type=_obj::pipeline; if(bl!=nullptr) cmds.push_back(bl); negated=false; bash_time=false; }
+  ~pipeline_t() { for(auto it: cmds) delete it; }
+  inline void add(block_t* bl) { this->cmds.push_back(bl); }
+  std::vector<block_t*> cmds;
 
   bool negated; // negated return value (! at start)
+  bool bash_time; // has bash time command
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); };
 };
 
 // CL
 
-class condlist : public _obj
+class condlist_t : public _obj
 {
 public:
-  condlist() { type=_obj::_condlist; parallel=false; }
-  condlist(pipeline* pl);
-  condlist(block* bl);
-  ~condlist() { for(auto it: pls) delete it; }
+  condlist_t() { type=_obj::condlist; parallel=false; }
+  condlist_t(pipeline_t* pl) { type=_obj::condlist; parallel=false; this->add(pl); }
+  condlist_t(block_t* bl);
+  ~condlist_t() { for(auto it: pls) delete it; }
 
   bool parallel; // & at the end
 
-  void add(pipeline* pl, bool or_op=false);
+  void add(pipeline_t* pl, bool or_op=false);
+
   // don't push_back here, use add() instead
-  std::vector<pipeline*> pls;
+  std::vector<pipeline_t*> pls;
   std::vector<bool> or_ops; // size of 1 less than pls, defines separator between pipelines
 
   void prune_first_cmd();
 
-  block* first_block();
-  cmd* first_cmd();
-  cmd* get_cmd(std::string const& cmdname);
+  block_t* first_block();
+  cmd_t* first_cmd();
+  cmd_t* get_cmd(std::string const& cmdname);
 
   void negate();
 
   std::string generate(int ind);
 };
 
-class list : public _obj
+class list_t : public _obj
 {
 public:
-  list() { type=_obj::_list; }
-  ~list() { for(auto it: cls) delete it; }
+  list_t() { type=_obj::list; }
+  list_t(condlist_t* in) { type=_obj::list; this->add(in); }
+  ~list_t() { for(auto it: cls) delete it; }
 
-  std::vector<condlist*> cls;
+  void clear() { for(auto it: cls) delete it; cls.resize(0); }
 
-  condlist* last_cond() { return cls[cls.size()-1]; }
+  std::vector<condlist_t*> cls;
+  inline void add(condlist_t* in) { cls.push_back(in); }
+
+  condlist_t* last_cond() { return cls[cls.size()-1]; }
+
+  void insert(uint32_t i, condlist_t* val);
+  void insert(uint32_t i, list_t const& lst);
 
   size_t size() { return cls.size(); }
-  condlist* operator[](uint32_t i) { return cls[i]; }
 
   std::string generate(int ind, bool first_indent);
   std::string generate(int ind) { return this->generate(ind, true); }
 };
 
-// class redir
-// {
-// public:
-//   enum redirtype { none, write, append, read, raw } ;
-//   redir(redirtype in=none) { type=in; }
-//   redirtype type;
-//   arg val;
-// };
-
 // block subtypes //
 
-class cmd : public block
+class cmd_t : public block_t
 {
 public:
-  cmd(arglist* in=nullptr) { type=_obj::block_cmd; args=in; }
-  ~cmd() {
+  cmd_t(arglist_t* in=nullptr) { type=_obj::block_cmd; args=in; is_cmdvar=false; }
+  ~cmd_t() {
     if(args!=nullptr) delete args;
-    for(auto it: var_assigns) delete it.second;
+    for(auto it: var_assigns) {
+      delete it.first;
+      delete it.second;
+    }
+    for(auto it: cmd_var_assigns) {
+      delete it.first;
+      delete it.second;
+    }
   }
 
   static const std::string empty_string;
 
-  std::string const& firstarg_string();
+  std::string const& arg_string(uint32_t n);
 
   size_t arglist_size();
 
-  void add_arg(arg* in);
+  void add(arg_t* in);
 
 
   // preceding var assigns
-  std::vector<std::pair<std::string,arg*>> var_assigns;
+  std::vector<std::pair<variable_t*,arg_t*>> var_assigns;
+
+  // is a cmdvar type
+  bool is_cmdvar;
+  // var assigns on cmdvar
+  std::vector<std::pair<variable_t*,arg_t*>> cmd_var_assigns;
 
   // check if cmd is this (ex: unset)
   bool is(std::string const& in);
   // for var assigns in special cmds (export, unset, read, local)
   bool is_argvar();
-  std::vector<subarg*> subarg_vars();
+  std::vector<subarg_t*> subarg_vars();
 
-  arglist* args;
+  // returns true if command performs env var changes
+  bool has_var_assign();
 
-  std::string generate(int ind);
+  arglist_t* args;
+
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
-class shmain : public block
+class shmain : public block_t
 {
 public:
-  shmain(list* in=nullptr) { type=_obj::block_main; lst=in; }
+  shmain(list_t* in=nullptr) { type=_obj::block_main; if(in == nullptr) lst = new list_t; else lst=in; }
   ~shmain() {
     if(lst!=nullptr) delete lst;
   }
 
-  bool is_dev_file() { return filename.substr(0,5) == "/dev/"; }
-
   void concat(shmain* in);
 
   std::string filename;
   std::string shebang;
-  list* lst;
+  list_t* lst;
 
   std::string generate(bool print_shebang=true, int ind=0);
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
-class subshell : public block
+class subshell_t : public block_t
 {
 public:
-  subshell(list* in=nullptr) { type=_obj::block_subshell; lst=in; }
-  ~subshell() {
+  subshell_t(list_t* in=nullptr) { type=_obj::block_subshell; lst=in; }
+  subshell_t(block_t* in) { type=_obj::block_subshell; lst=new list_t(new condlist_t(in)); }
+  ~subshell_t() {
     if(lst!=nullptr) delete lst;
   }
 
-  cmd* single_cmd();
+  cmd_t* single_cmd();
 
-  list* lst;
+  list_t* lst;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
-class brace : public block
+class brace_t : public block_t
 {
 public:
-  brace(list* in=nullptr) { type=_obj::block_brace; lst=in; }
-  ~brace() {
+  brace_t(list_t* in=nullptr) { type=_obj::block_brace; lst=in; }
+  ~brace_t() {
     if(lst!=nullptr) delete lst;
   }
 
-  cmd* single_cmd();
+  cmd_t* single_cmd();
 
-  list* lst;
+  list_t* lst;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
-class function : public block
+class function_t : public block_t
 {
 public:
-  function(list* in=nullptr) { type=_obj::block_function; lst=in; }
-  ~function() {
+  function_t(list_t* in=nullptr) { type=_obj::block_function; lst=in; }
+  ~function_t() {
     if(lst!=nullptr) delete lst;
   }
 
   std::string name;
-  list* lst;
+  list_t* lst;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
-class case_block : public block
+class case_t : public block_t
 {
 public:
-  case_block(arg* in=nullptr) { type=_obj::block_case; carg=in; }
-  ~case_block() {
+  case_t(arg_t* in=nullptr) { type=_obj::block_case; carg=in; }
+  ~case_t() {
     if(carg!=nullptr) delete carg;
     for( auto cit : cases )
     {
@@ -355,17 +482,18 @@ public:
     }
   }
 
-  arg* carg;
-  std::vector< std::pair<std::vector<arg*>, list*> > cases;
+  arg_t* carg;
+  std::vector< std::pair<std::vector<arg_t*>, list_t*> > cases;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
-class if_block : public block
+class if_t : public block_t
 {
 public:
-  if_block() { type=_obj::block_if; else_lst=nullptr; }
-  ~if_block() {
+  if_t() { type=_obj::block_if; else_lst=nullptr; }
+  ~if_t() {
     for(auto ifb: blocks)
     {
       if(ifb.first!=nullptr) delete ifb.first;
@@ -374,106 +502,181 @@ public:
     if(else_lst!=nullptr) delete else_lst;
   }
 
-  std::vector< std::pair<list*,list*> > blocks;
+  std::vector< std::pair<list_t*,list_t*> > blocks;
 
-  list* else_lst;
+  list_t* else_lst;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
-class for_block : public block
+class for_t : public block_t
 {
 public:
-  for_block(std::string const& name="", arglist* args=nullptr, list* lst=nullptr) { type=_obj::block_for; varname=name; iter=args; ops=lst; }
-  ~for_block() {
+  for_t(variable_t* in=nullptr, arglist_t* args=nullptr, list_t* lst=nullptr, bool ii=false) { type=_obj::block_for; var=in; iter=args; ops=lst; in_val=ii; }
+  ~for_t() {
     if(iter!=nullptr) delete iter;
     if(ops!=nullptr) delete ops;
+    if(var!=nullptr) delete var;
   }
 
-  std::string varname;
+  variable_t* var;
 
-  arglist* iter;
-  list* ops;
+  arglist_t* iter;
+  list_t* ops;
 
-  std::string generate(int ind);
+  bool in_val;
+
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
-class while_block : public block
+class while_t : public block_t
 {
 public:
-  while_block(list* a=nullptr, list* b=nullptr) { type=_obj::block_while; cond=a; ops=b; }
-  ~while_block() {
+  while_t(list_t* a=nullptr, list_t* b=nullptr) { type=_obj::block_while; cond=a; ops=b; }
+  ~while_t() {
     if(cond!=nullptr) delete cond;
     if(ops!=nullptr) delete ops;
   }
 
-  condlist* real_condition() { return cond->last_cond(); }
+  condlist_t* real_condition() { return cond->last_cond(); }
 
-  list* cond;
-  list* ops;
+  list_t* cond;
+  list_t* ops;
 
-  std::string generate(int ind);
+  std::string generate(int ind, generate_context* ctx);
+  std::string generate(int ind) { return this->generate(ind, nullptr); }
 };
 
 // Subarg subtypes //
 
-class string_subarg : public subarg
+class subarg_string_t : public subarg_t
 {
 public:
-  string_subarg(std::string const& in="") { type=_obj::subarg_string; val=in; }
-  ~string_subarg() {;}
+  subarg_string_t(std::string const& in="") { type=_obj::subarg_string; val=in; }
+  ~subarg_string_t() {;}
 
   std::string val;
 
   std::string generate(int ind) { return val; }
 };
 
-class variable_subarg : public subarg
+class subarg_variable_t : public subarg_t
 {
 public:
-  variable_subarg(std::string const& in="") { type=_obj::subarg_variable; varname=in; }
-  ~variable_subarg() {;}
+  subarg_variable_t(variable_t* in=nullptr, bool inq=false) { type=_obj::subarg_variable; var=in; quoted=inq; }
+  ~subarg_variable_t() {
+    if(var!=nullptr) delete var;
+  }
 
-  std::string varname;
+  variable_t* var;
 
-  std::string generate(int ind) { return "$" + varname; }
+  std::string generate(int ind) { return "$" + var->generate(ind); }
 };
 
-class arithmetic_subarg : public subarg
+class subarg_arithmetic_t : public subarg_t
 {
 public:
-  arithmetic_subarg() { type=_obj::subarg_arithmetic; }
-  ~arithmetic_subarg() {;}
+  subarg_arithmetic_t(arithmetic_t* a=nullptr) { type=_obj::subarg_arithmetic; arith=a; }
+  ~subarg_arithmetic_t() {
+    if(arith!=nullptr) delete arith;
+  }
+
+  arithmetic_t* arith;
+
+  std::string generate(int ind);
+};
+
+class subarg_subshell_t : public subarg_t
+{
+public:
+  subarg_subshell_t(subshell_t* in=nullptr, bool inq=false, bool is_backtick=false) { type=_obj::subarg_subshell; sbsh=in; quoted=inq; backtick=is_backtick; }
+  ~subarg_subshell_t() { if(sbsh != nullptr) delete sbsh; }
+
+  subshell_t* sbsh;
+  bool backtick;
+
+  std::string generate(int ind);
+};
+
+class subarg_procsub_t : public subarg_t
+{
+public:
+  subarg_procsub_t() { type=_obj::subarg_procsub; sbsh=nullptr; is_output=false; }
+  subarg_procsub_t(bool output, subshell_t* in) { type=_obj::subarg_procsub; sbsh=in; is_output=output; }
+  ~subarg_procsub_t() { if(sbsh!=nullptr) delete sbsh; }
+
+  bool is_output;
+  subshell_t* sbsh;
+
+  std::string generate(int ind);
+};
+
+// Arithmetic subtypes //
+
+class arithmetic_operation_t : public arithmetic_t
+{
+public:
+  arithmetic_operation_t(std::string op="", arithmetic_t* a=nullptr, arithmetic_t* b=nullptr, bool pre=false) { type=_obj::arithmetic_operation; oper=op; val1=a; val2=b; precedence=pre; }
+  ~arithmetic_operation_t() {
+    if(val1 != nullptr) delete val1;
+    if(val2 != nullptr) delete val2;
+  }
+
+  std::string oper;
+  bool precedence;
+  arithmetic_t *val1, *val2;
+  std::string generate(int ind);
+};
+
+class arithmetic_subshell_t : public arithmetic_t
+{
+public:
+  arithmetic_subshell_t(subshell_t* a=nullptr) { type=_obj::arithmetic_subshell; sbsh=a; }
+  ~arithmetic_subshell_t() {
+    if(sbsh!=nullptr) delete sbsh;
+  }
+
+  subshell_t* sbsh;
+
+  std::string generate(int ind);
+};
+
+class arithmetic_parenthesis_t : public arithmetic_t
+{
+public:
+  arithmetic_parenthesis_t(arithmetic_t* a=nullptr) { type=_obj::arithmetic_parenthesis; val=a; }
+  ~arithmetic_parenthesis_t() {
+    if(val!=nullptr) delete val;
+  }
+
+  arithmetic_t* val;
+
+  std::string generate(int ind);
+};
+
+class arithmetic_number_t : public arithmetic_t
+{
+public:
+  arithmetic_number_t(std::string const& a) { type=_obj::arithmetic_number; val=a; }
 
   std::string val;
 
-  std::string generate(int ind) { return "$(("+val+"))"; }
+  std::string generate(int ind) { return val; }
 };
 
-class subshell_subarg : public subarg
+class arithmetic_variable_t : public arithmetic_t
 {
 public:
-  subshell_subarg(subshell* in=nullptr, bool inq=false) { type=_obj::subarg_subshell; sbsh=in; quoted=inq; }
-  ~subshell_subarg() { if(sbsh != nullptr) delete sbsh; }
+  arithmetic_variable_t(variable_t* in=nullptr) { type=_obj::arithmetic_variable; var=in; }
+  ~arithmetic_variable_t() {
+    if(var!=nullptr) delete var;
+  }
 
-  subshell* sbsh;
-  bool quoted;
+  variable_t* var;
 
   std::string generate(int ind);
 };
 
-class manipulation_subarg : public subarg
-{
-public:
-  manipulation_subarg(arg* in=nullptr) { type=_obj::subarg_manipulation; size=false; manip=in; }
-  ~manipulation_subarg() { if(manip!=nullptr) delete manip; }
-
-  bool size;
-  std::string varname;
-  arg* manip;
-
-  std::string generate(int ind);
-};
-
-
 #endif //STRUC_HPP

include/struc_helper.hpp

@@ -0,0 +1,46 @@
+#ifndef STRUC_HELPER_HPP
+#define STRUC_HELPER_HPP
+
+#include "struc.hpp"
+
+// makers
+arg_t* make_arg(std::string const& in);
+
+cmd_t* make_cmd(std::vector<const char*> const& args);
+cmd_t* make_cmd(std::vector<std::string> const& args);
+cmd_t* make_cmd(std::vector<arg_t*> const& args);
+cmd_t* make_cmd(std::string const& in);
+
+pipeline_t* make_pipeline(std::vector<block_t*> const& bls);
+pipeline_t* make_pipeline(std::string const& in);
+
+condlist_t* make_condlist(std::string const& in);
+list_t* make_list(std::string const& in);
+
+block_t* make_block(std::string const& in);
+
+// copy
+arg_t* copy(arg_t* in);
+variable_t* copy(variable_t* in);
+
+// testers
+bool arg_has_char(char c, arg_t* in);
+bool possibly_expands(arg_t* in);
+bool possibly_expands(arglist_t* in);
+
+// modifiers
+void force_quotes(arg_t* in);
+void add_quotes(arg_t* in);
+
+cmd_t* make_printf(arg_t* in);
+inline cmd_t* make_printf_variable(std::string const& name) {
+  return make_printf(new arg_t(new subarg_variable_t(new variable_t(name))));
+}
+
+arithmetic_t* make_arithmetic(arg_t* a);
+arithmetic_t* make_arithmetic(arg_t* arg1, std::string op, arg_t* arg2);
+
+// operators
+inline bool operator==(arg_t a, std::string const& b) { return a.equals(b); }
+
+#endif //STRUC_HELPER_HPP

include/util.hpp

@@ -12,15 +12,16 @@
 #include <functional>
 #include <regex>
 
-#include <ztd/filedat.hpp>
-
 #include "struc.hpp"
 
 extern std::string indenting_string;
 
+std::string cut_last(std::string const& in, char c);
 std::string basename(std::string const& in);
 std::string dirname(std::string const& in);
 
+inline bool is_dev_file(std::string const& filename) { return filename.substr(0,5) == "/dev/"; }
+
 std::string indent(int n);
 
 std::vector<std::string> split(std::string const& in, const char* splitters);
@@ -43,6 +44,11 @@ std::string strf( const std::string& format, Args ... args )
     return std::string( buf.get(), buf.get() + size - 1 ); // We don't want the '\0' inside
 }
 
+template<class T, typename ... Args>
+std::vector<T> make_vector(Args ... args)
+{
+  return std::vector<T>( { args... } );
+}
 
 template <class KEY, class VAL>
 std::vector<std::pair<KEY, VAL>> sort_by_value(std::map<KEY,VAL> const& in)
@@ -120,6 +126,32 @@ void concat_sets(std::set<T>& a, std::set<T> const& b)
   }
 }
 
+template <class T>
+void exclude_sets(std::set<T>& a, std::set<T> const& b)
+{
+  for(auto it: b)
+  {
+    auto t = a.find(it);
+    if(t != a.end())
+      a.erase(t);
+  }
+}
+
+template <class T>
+bool is_in_vector(T el, std::vector<T> vec)
+{
+  for(auto it: vec)
+    if(it == el)
+      return true;
+  return false;
+}
+
+template <class T>
+bool is_in_set(T el, std::set<T> ss)
+{
+  return ss.find(el) != ss.end();
+}
+
 std::set<std::string> prune_matching(std::set<std::string>& in, std::regex re);
 
 std::string delete_brackets(std::string const& in);
@@ -129,10 +161,8 @@ std::string concatargs(std::vector<std::string> const& args);
 int _exec(std::string const& bin, std::vector<std::string> const& args);
 
 std::string stringReplace(std::string subject, const std::string& search, const std::string& replace);
+std::string escape_chars(std::string subject, const char* chars);
 
-void printFormatError(ztd::format_error const& e, bool print_line=true);
-void printErrorIndex(const char* in, const int index, const std::string& message, const std::string& origin, bool print_line=true);
-
-int execute(shmain* sh, std::vector<std::string>& args);
+void printFormatError(format_error const& e, bool print_line=true);
 
 #endif //UTIL_HPP

include/version.h

@@ -1,6 +1,6 @@
 #ifndef VERSION_H
 #define VERSION_H
 
-#define VERSION_STRING "v0.2.1"
+#define VERSION_STRING "v1.5"
 
 #endif //VERSION_H

run-tests.sh

@@ -0,0 +1,233 @@
+#!/bin/bash
+
+bin=${1-./lxsh}
+
+echo_red()
+{
+  printf "\033[1;31m%s\033[0m\n" "$*"
+  exit 1
+}
+
+err=0
+
+# $1 = file , $2 = extra print
+# _LXSH_OPT : lxsh options
+compile_test()
+{
+  printf "%s%s: " "$1" "$2"
+  if errout=$($bin $_LXSH_OPT "$1" 2>&1 >/dev/null)
+  then
+    echo "Ok"
+  else
+    echo_red "Error"
+    echo "$errout"
+    return 1
+  fi
+}
+
+# $1 = runner , $2 = file , $3 = extra print , $4 = runtime for lxsh
+# _LXSH_OPT : lxsh options
+exec_test()
+{
+  run=$1
+  lxshrun=${4-$run}
+  ret1=$($run "$2")
+  stat1=$?
+  ret2=$($bin $_LXSH_OPT "$2" | $lxshrun)
+  stat2=$?
+  printf "%s%s: " "$2" "$3"
+  if [ "$ret1" = "$ret2" ] && [ $stat1 -eq $stat2 ]
+  then echo "Ok"
+  else
+    echo_red "Error"
+    echo ">> original stat $stat1
+$ret1
+>> compiled stat $stat2
+$ret2"
+    return 1
+  fi
+}
+
+size_test()
+{
+  shebang=$(head -n1 "$1" | grep '^#!')
+  c1=$($bin --no-shebang -m "$1" | wc -c)
+  c2=$($bin -m "$1" | shfmt -mn | wc -c)
+  printf "%s%s: " "$1" "$2"
+  if [ $c1 -lt $c2 ]
+  then echo "Ok"
+  else
+    echo_red "Too big"
+    return 1
+  fi
+}
+
+# $1 = file , $2 = extra print , $3 = list , $@ = run options
+list_test()
+{
+  printf "%s%s: " "$1" "$2"
+  file=$1
+  varlist=$3
+  shift 3
+  diffout=$(diff <($bin "$file" "$@" | sort -k2) <(echo "$varlist" | sed '/^$/d' | sort -k2) )
+  if [ -z "$diffout" ] ; then
+    echo "Ok"
+  else
+    echo_red "Variable mismatch"
+    echo "$diffout"
+    return 1
+  fi
+}
+
+resolve="test/include.sh test/resolve.sh"
+exec_exclude="test/prompt.sh $resolve"
+
+
+echo "
+============
+|    sh    |
+============
+"
+
+echo "== Parse =="
+for I in test/*.sh
+do
+  compile_test "$I" || err=$((err+1))
+done
+
+echo "== Exec =="
+for I in $( echo test/*.sh $exec_exclude | tr -s ' \n' '\n' | sort | uniq -u )
+do
+  exec_test sh "$I" || err=$((err+1))
+  _LXSH_OPT=-M exec_test sh "$I" " (minify)" || err=$((err+1))
+done
+
+echo "== Size =="
+for I in test/*.sh
+do
+  size_test "$I" || err=$((err+1))
+done
+
+echo "== Resolve =="
+
+for I in $resolve
+do
+  printf "%s: " "$I"
+  if errmsg=$($bin "$I" | sh 2>&1 >/dev/null) && [ -z "$errmsg" ]
+  then echo "Ok"
+  else
+    echo_red "Error"
+    echo ">> stderr
+$errmsg"
+    err=$((err+1))
+  fi
+done
+
+varlist="
+2 nul
+2 ABCD
+1 AYE
+1 BAR
+3 FOO
+2 TATA
+1 TITI
+4 TOTO
+1 TUTU
+4 somevar
+"
+
+vardefs="
+1 ABCD
+1 BAR
+2 FOO
+1 TATA
+1 TOTO
+1 TUTU
+1 nul
+2 somevar
+"
+
+varcalls="
+1 AYE
+1 ABCD
+1 FOO
+1 TATA
+1 TITI
+3 TOTO
+1 nul
+2 somevar
+"
+
+varlist_used="
+1 AYE
+2 ABCD
+3 FOO
+2 TATA
+1 TITI
+4 TOTO
+2 nul
+4 somevar
+"
+
+echo "== Variables =="
+{
+  list_test test/var.sh " (list)" "$varlist" --list-var || err=$((err+1))
+  list_test test/var.sh " (list-def)" "$vardefs" --list-var-def || err=$((err+1))
+  list_test test/var.sh " (list-call)" "$varcalls" --list-var-call || err=$((err+1))
+  list_test test/var.sh " (remove unused)" "$varlist_used" --remove-unused --list-var || err=$((err+1))
+}
+
+fctlist="
+1 toto
+1 tata
+"
+
+fctlist_used="
+1 toto
+"
+
+cmdlist="
+2 echo
+1 toto
+"
+
+echo "== Functions =="
+{
+  list_test test/fct.sh " (list-fct)" "$fctlist" --list-fct || err=$((err+1))
+  list_test test/fct.sh " (list-cmd)" "$cmdlist" --list-cmd || err=$((err+1))
+  list_test test/fct.sh " (remove unused)" "$fctlist_used" --remove-unused --list-fct || err=$((err+1))
+}
+
+echo "
+============
+|   bash   |
+============
+"
+
+echo "== Parse =="
+for I in test/*.bash
+do
+  compile_test "$I" || err=$((err+1))
+done
+
+echo "== Exec =="
+for I in test/*.bash
+do
+  exec_test bash "$I" || err=$((err+1))
+  _LXSH_OPT=-m exec_test bash "$I" " (minify)" || err=$((err+1))
+done
+
+echo "== Size =="
+for I in test/*.bash
+do
+  size_test "$I" || err=$((err+1))
+done
+
+echo "== Debashify =="
+for I in test/{debashify.bash,array.bash,echo.bash}
+do
+  _LXSH_OPT=--debashify exec_test bash "$I" "" sh || err=$((err+1))
+  _LXSH_OPT="-m --debashify" exec_test bash "$I" " (minify)" sh || err=$((err+1))
+done
+
+exit $err

shellcode/array_create.sh

@@ -0,0 +1,7 @@
+_lxsh_array_create() {
+  printf "%s" "$1"
+  shift 1 2>/dev/null || return
+  for N ; do
+    printf "\t%s" "$N"
+  done
+}

shellcode/array_get.sh

@@ -0,0 +1,7 @@
+_lxsh_array_get() {
+  if [ "$2" = "*" ] || [ "$2" = "@" ] ; then
+    printf "%s" "$1" | tr '\t' ' '
+  else
+    printf "%s" "$1" | cut -f$(($2+1))
+  fi
+}

shellcode/array_set.sh

@@ -0,0 +1,6 @@
+_lxsh_array_set()
+{
+  [ "$2" -gt 0 ] && printf "%s\t" "$(printf "%s" "$1" | cut -f1-$2)"
+  printf "%s" "$3"
+  [ "$2" -lt $(printf "%s" "$1" | tr -dc '\t' | wc -c) ] && { printf "\t" ; printf "%s" "$1"|cut -f$(($2+2))-; }
+}

shellcode/map_create.sh

@@ -0,0 +1,6 @@
+_lxsh_map_create() {
+  for I
+  do
+    printf "%s]%s\n" "$(echo "$I" | cut -d']' -f1 | cut -d '[' -f2)" "$(echo "$I" | cut -d '=' -f2-)"
+  done
+}

shellcode/map_get.sh

@@ -0,0 +1,7 @@
+_lxsh_map_get() {
+  if [ "$2" = \* ] || [ "$2" = @ ] ; then
+    printf "%s" "$(printf "%s" "$1" | sort | cut -d ']' -f2-)" | tr '\n' ' '
+  else
+    printf "%s\n" "$1" | grep "^$2\]" | cut -d ']' -f2-
+  fi
+}

shellcode/map_set.sh

@@ -0,0 +1,6 @@
+_lxsh_map_set() {
+  printf "%s\n" "$1" | grep -v "^$2\]"
+  if [ -n "$3" ] ; then
+    printf "%s]%s\n" "$2" "$3"
+  fi
+}

shellcode/random.sh

@@ -0,0 +1,3 @@
+_lxsh_random() {
+  printf %d "0x$(head -c"${1-2}" </dev/urandom | od -A n -vt x1 | tr -d ' ')"
+}

shellcode/random_string.sh

@@ -0,0 +1,3 @@
+_lxsh_random_string() {
+  env LC_CTYPE=C tr -dc 'a-zA-Z0-9' </dev/urandom | head -c "${1-20}"
+}

shellcode/random_tmpfile.sh

@@ -0,0 +1,3 @@
+_lxsh_random_tmpfile() {
+  echo "${TMPDIR-/tmp}/$1$(_lxsh_random_string $2)"
+}

src/exec.cpp

@@ -0,0 +1,275 @@
+#include "exec.hpp"
+
+#include <unistd.h>
+#include <sys/wait.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <signal.h>
+
+#include "g_shellcode.h"
+
+#include "util.hpp"
+#include "parse.hpp"
+#include "debashify.hpp"
+#include "resolve.hpp"
+#include "recursive.hpp"
+#include "shellcode.hpp"
+
+#define PIPE_READ  0
+#define PIPE_WRITE 1
+
+std::vector<condlist_t*> do_include_exec(condlist_t* cmd, parse_context ctx, FILE* fd)
+{
+  std::vector<condlist_t*> ret;
+
+  std::string dir;
+  auto incs=do_include_raw(cmd, ctx, &dir);
+
+  for(auto it: incs)
+  {
+    parse_exec(fd, make_context(ctx, it.second, it.first));
+  }
+  // cd back
+  _cd(dir);
+
+  return ret;
+}
+
+// if first is nullptr: is a string
+std::vector<condlist_t*> do_resolve_exec(condlist_t* cmd, parse_context ctx, FILE* fd)
+{
+  std::vector<condlist_t*> ret;
+
+  std::pair<std::string,std::string> p;
+  try
+  {
+    // get
+    std::string dir;
+    p=do_resolve_raw(cmd, ctx, &dir);
+    // do parse
+    parse_exec(fd, make_context(ctx, p.second, p.first));
+    // cd back
+    _cd(dir);
+  }
+  catch(format_error& e)
+  {
+    throw format_error(e.what(), '`'+p.first+'`', e.data(), e.where());
+  }
+
+  return ret;
+}
+
+// -- OBJECT CALLS --
+
+bool resolve_condlist_exec(condlist_t* in, parse_context ctx, FILE* fd)
+{
+  cmd_t* tc = in->first_cmd();
+  if(tc == nullptr)
+    return false;
+
+  std::string const& strcmd=tc->arg_string(0);
+
+  if(g_include && strcmd == "%include")
+  {
+    do_include_exec(in, ctx, fd);
+    return true;
+  }
+  else if(g_resolve && strcmd == "%resolve")
+  {
+    do_resolve_exec(in, ctx, fd);
+    return true;
+  }
+  return false;
+}
+
+
+bool resolve_exec(condlist_t* in, parse_context ctx, FILE* fd)
+{
+  if(!resolve_condlist_exec(in, ctx, fd))
+  {
+    resolve(in, ctx);
+    return false;
+  }
+  return true;
+}
+
+char byte_to_char(uint8_t in)
+{
+  uint8_t t = in&0b00111111; // equiv to %64
+  if(t < 26)
+    return t+'a';
+  if(t < 52)
+    return (t-26)+'A';
+  if(t < 62)
+    return (t-52)+'0';
+  if(t == 62)
+    return '-';
+  return '_';
+}
+
+std::string gettmpdir()
+{
+  std::string tmpdir;
+  char* tbuf = getenv("TMPDIR");
+  if(tbuf != NULL)
+    tmpdir = tbuf;
+  if(tmpdir == "")
+    tmpdir = "/tmp";
+  return tmpdir;
+}
+
+// random string of size 20
+std::string random_string()
+{
+  // get system random seed
+  FILE* f = fopen("/dev/urandom", "r");
+  if(!f)
+    throw std::runtime_error("Cannot open stream to /dev/urandom");
+  uint8_t buffer[20];
+  size_t r = fread(buffer, 20, 1, f);
+  fclose(f);
+  if(r<=0)
+    throw std::runtime_error("Cannot read from /dev/urandom");
+
+  std::string ret;
+  for(uint8_t i=0; i<20; i++)
+    ret += byte_to_char(buffer[i]);
+
+  return ret;
+}
+
+void parse_exec(FILE* fd, parse_context ctx)
+{
+  ctx.i=skip_unread(ctx);
+
+  debashify_params debash_params;
+  list_t* t_lst=new list_t;
+  if(t_lst == nullptr)
+    throw std::runtime_error("Alloc error");
+  while(ctx.i<ctx.size)
+  {
+    auto pp=parse_condlist(ctx);
+    ctx=pp.second;
+    if(ctx.has_errored)
+    {
+      parse_list_until(ctx);
+      throw std::runtime_error("Aborting due to previous errors");
+    }
+    t_lst->add(pp.first);
+    if(g_resolve || g_include)
+    {
+      if(resolve_exec(t_lst->cls[0], ctx, fd))
+      {
+        t_lst->clear();
+        continue;
+      }
+    }
+    if(options["debashify"])
+      debashify(t_lst, &debash_params);
+
+
+    std::string gen=t_lst->generate(0);
+    t_lst->clear();
+
+    fprintf(fd, "%s", gen.c_str());
+
+    if(ctx.i < ctx.size)
+    {
+      if(ctx[ctx.i] == '#')
+        ; // skip here
+      else if(is_in(ctx[ctx.i], COMMAND_SEPARATOR))
+        ctx.i++; // skip on next char
+      else if(is_in(ctx[ctx.i], CONTROL_END))
+      {
+        format_error(strf("Unexpected token: '%c'", ctx[ctx.i]), ctx);
+        return;
+      }
+
+      ctx.i = skip_unread(ctx);
+    }
+  }
+  delete t_lst;
+}
+
+pid_t forkexec(const char* bin, char *const args[])
+{
+  pid_t child_pid;
+  if((child_pid = vfork()) == -1)
+  {
+    throw std::runtime_error("fork() failed");
+  }
+  if (child_pid == 0) // child process
+  {
+    execv(bin, args);
+    throw std::runtime_error("execv() failed");
+  }
+  else // main process
+  {
+    return child_pid;
+  }
+}
+
+int wait_pid(pid_t pid)
+{
+  int stat;
+  while (waitpid(pid, &stat, 0) == -1)
+  {
+    if (errno != EINTR)
+    {
+      stat = -1;
+      break;
+    }
+  }
+
+  return WEXITSTATUS(stat);
+}
+
+int exec_process(std::string const& runtime, std::vector<std::string> const& args, parse_context ctx)
+{
+  std::vector<std::string> strargs = split(runtime, " \t");
+  std::vector<char*> runargs;
+
+  std::string fifopath=gettmpdir();
+  fifopath+="/lxshfiforun_";
+  fifopath+=random_string();
+
+  if(mkfifo(fifopath.c_str(), 0700)<0)
+    throw std::runtime_error("Cannot create fifo "+fifopath);
+
+  pid_t pid=0;
+  FILE* ffd=0;
+  try
+  {
+
+    for(uint32_t i=0; i<strargs.size(); i++)
+      runargs.push_back((char*) strargs[i].c_str());
+    runargs.push_back((char*) fifopath.c_str());
+    for(uint32_t i=0; i<args.size(); i++)
+      runargs.push_back((char*) args[i].c_str());
+    runargs.push_back(NULL);
+
+    pid = forkexec(runargs[0], runargs.data());
+    ffd = fopen(fifopath.c_str(), "w");
+    if(options["debashify"])
+    {
+      for(auto it: lxsh_array_fcts)
+        fprintf(ffd, "%s\n", it.second.code);
+    }
+    for(auto it: lxsh_extend_fcts)
+      fprintf(ffd, "%s\n", it.second.code);
+    parse_exec(ffd, ctx);
+  }
+  catch(std::runtime_error& e)
+  {
+    if(pid != 0)
+      kill(pid, SIGINT);
+    fclose(ffd);
+    unlink(fifopath.c_str());
+    throw e;
+  }
+
+  fclose(ffd);
+  unlink(fifopath.c_str());
+
+  return wait_pid(pid);
+}

src/generate.cpp

@@ -6,6 +6,9 @@
 #include "options.hpp"
 #include "parse.hpp"
 
+// global
+bool prev_is_heredoc=false;
+
 bool is_sub_special_cmd(std::string in)
 {
   return in == "%include_sub" || in == "%resolve_sub";
@@ -13,13 +16,13 @@ bool is_sub_special_cmd(std::string in)
 
 std::string indented(std::string const& in, uint32_t ind)
 {
-  if(!opt_minimize)
+  if(!opt_minify)
     return indent(ind) + in;
   else
     return in;
 }
 
-std::string arg::generate(int ind)
+std::string arg_t::generate(int ind)
 {
   std::string ret;
   for(auto it: sa)
@@ -29,7 +32,7 @@ std::string arg::generate(int ind)
   return ret;
 }
 
-std::string arglist::generate(int ind)
+std::string arglist_t::generate(int ind)
 {
   std::string ret;
 
@@ -44,7 +47,7 @@ std::string arglist::generate(int ind)
   return ret;
 }
 
-std::string pipeline::generate(int ind)
+std::string pipeline_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -53,84 +56,110 @@ std::string pipeline::generate(int ind)
 
   if(negated)
     ret += "! ";
-  ret += cmds[0]->generate(ind);
+
+  if(bash_time)
+    ret += "time ";
+
+  ret += cmds[0]->generate(ind, ctx);
   for(uint32_t i=1 ; i<cmds.size() ; i++)
   {
-    ret += opt_minimize ? "|" : " | " ;
-    ret += cmds[i]->generate(ind);
+    ret += opt_minify ? "|" : " | " ;
+    ret += cmds[i]->generate(ind, ctx);
   }
 
   return ret;
 }
 
-std::string condlist::generate(int ind)
+std::string condlist_t::generate(int ind)
 {
   std::string ret;
   if(pls.size() <= 0)
     return "";
-  ret += pls[0]->generate(ind);
+  generate_context ctx;
+  ret += pls[0]->generate(ind, &ctx);
   for(uint32_t i=0 ; i<pls.size()-1 ; i++)
   {
     if(or_ops[i])
-      ret += opt_minimize ? "||" : " || ";
+      ret += opt_minify ? "||" : " || ";
     else
-      ret += opt_minimize ? "&&" : " && ";
-    ret += pls[i+1]->generate(ind);
+      ret += opt_minify ? "&&" : " && ";
+    ret += pls[i+1]->generate(ind, &ctx);
   }
+  prev_is_heredoc=false;
   if(ret=="")
     return "";
-  if(parallel)
+  if(ctx.here_document != nullptr)
   {
-    ret += opt_minimize ? "&" : " &\n";
+    if(parallel)
+      ret += '&';
+    ret += '\n';
+    ret += ctx.here_document->generate(0);
+    ret += '\n';
+    prev_is_heredoc=true;
+  }
+  else if(parallel)
+  {
+    ret += opt_minify ? "&" : " &\n";
   }
   else
     ret += '\n';
   return ret;
 }
 
-std::string list::generate(int ind, bool first_indent)
+std::string list_t::generate(int ind, bool first_indent)
 {
   std::string ret;
   if(cls.size() <= 0)
     return "";
 
+  std::string next;
   for(uint32_t i=0; i<cls.size(); i++)
   {
     if(first_indent)
     {
-      ret += indented(cls[i]->generate(ind), ind);
+      next = indented(cls[i]->generate(ind), ind);
     }
     else
     {
       first_indent=true;
-      ret += cls[i]->generate(ind);
+      next = cls[i]->generate(ind);
     }
+    if(ret[ret.size()-1] == '&' && next.size()>0 && is_in(next[0], "<>"))
+      ret += '\n';
+    ret += next;
   }
   return ret;
 }
 
-std::string redirect::generate(int ind)
+std::string redirect_t::generate(int ind)
 {
   std::string ret=op;
   if(target!=nullptr)
   {
-    if(!opt_minimize)
+    std::string targetret=target->generate(0);
+    if(!(opt_minify && !is_in(targetret[0], "<>")))
       ret += ' ';
-    ret += target->generate(0);
+    ret += targetret;
   }
   return ret;
 }
 
 // BLOCK
 
-std::string block::generate_redirs(int ind, std::string const& _str)
+std::string block_t::generate_redirs(int ind, std::string const& _str, generate_context* ctx=nullptr)
 {
   std::string ret=" ";
   bool previous_isnt_num = _str.size()>0 && !is_num(_str[_str.size()-1]);
   for(auto it: redirs)
   {
+    if(ctx != nullptr && it->here_document != nullptr)
+    {
+      if(ctx->here_document != nullptr)
+        throw std::runtime_error("Unsupported generation of concurrent here documents");
+      ctx->here_document = it->here_document;
+    }
     std::string _r = it->generate(0);
-    if(opt_minimize && _r.size() > 0 && !is_num(_r[0]) && previous_isnt_num)
+    if(opt_minify && _r.size() > 0 && !is_num(_r[0]) && previous_isnt_num)
       ret.pop_back(); // remove one space if possible
     ret += _r + ' ';
     previous_isnt_num = ret.size()>1 && !is_num(ret[ret.size()-2]);
@@ -139,7 +168,7 @@ std::string block::generate_redirs(int ind, std::string const& _str)
   return ret;
 }
 
-std::string if_block::generate(int ind)
+std::string if_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -149,7 +178,7 @@ std::string if_block::generate(int ind)
     if(i==0)
       ret += "if";
     else
-      ret += "elif";
+      ret += indented("elif", ind);
 
     if(blocks[i].first->size()==1)
       ret += ' ' + blocks[i].first->generate(ind+1, false);
@@ -169,29 +198,32 @@ std::string if_block::generate(int ind)
 
   ret += indented("fi", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string for_block::generate(int ind)
+std::string for_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
-  ret += "for "+varname;
-  if(iter != nullptr)
-    ret += " in " + iter->generate(ind);
+  ret += "for "+var->generate(ind);
+  if(in_val) {
+    ret += " in";
+    if(iter != nullptr)
+      ret += " " + iter->generate(ind);
+  }
   ret += '\n';
   ret += indented("do\n", ind);
   ret += ops->generate(ind+1);
   ret += indented("done", ind);
 
-  if(opt_minimize && ret.size()>1 && !is_alpha(ret[ret.size()-2]))
+  if(opt_minify && ret.size()>1 && !is_alpha(ret[ret.size()-2]))
     ret.pop_back();
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string while_block::generate(int ind)
+std::string while_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -205,30 +237,30 @@ std::string while_block::generate(int ind)
   ret += ops->generate(ind+1);
   ret += indented("done", ind);
 
-  if(opt_minimize && ret.size()>1 && !is_alpha(ret[ret.size()-2]))
+  if(opt_minify && ret.size()>1 && !is_alpha(ret[ret.size()-2]))
     ret.pop_back();
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string subshell::generate(int ind)
+std::string subshell_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
   // open subshell
   ret += '(';
-  if(!opt_minimize) ret += '\n';
+  if(!opt_minify) ret += '\n';
   // commands
   ret += lst->generate(ind+1);
-  if(opt_minimize && ret.size()>1)
+  if(opt_minify && ret.size()>1)
     ret.pop_back(); // ) can be right after command
   // close subshell
   ret += indented(")", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string shmain::generate(int ind)
+std::string shmain::generate(int ind, generate_context* ctx)
 {
   return this->generate(false, ind);
 }
@@ -238,14 +270,13 @@ std::string shmain::generate(bool print_shebang, int ind)
   if(print_shebang && shebang!="")
     ret += shebang + '\n';
   ret += lst->generate(ind);
-  if( opt_minimize && ret[ret.size()-1] == '\n')
+  if( opt_minify && ret[ret.size()-1] == '\n')
     ret.pop_back();
 
-  ret += generate_redirs(ind, ret);
   return ret;
 }
 
-std::string brace::generate(int ind)
+std::string brace_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
 
@@ -253,26 +284,26 @@ std::string brace::generate(int ind)
   ret += lst->generate(ind+1);
   ret += indented("}", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string function::generate(int ind)
+std::string function_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
   // function definition
   ret += name + "()";
-  if(!opt_minimize) ret += '\n';
+  if(!opt_minify) ret += '\n';
   // commands
   ret += indented("{\n", ind);
   ret += lst->generate(ind+1);
   ret += indented("}", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string case_block::generate(int ind)
+std::string case_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
   ret += "case " + carg->generate(ind) + " in\n";
@@ -286,67 +317,190 @@ std::string case_block::generate(int ind)
       ret += it->generate(ind) + '|';
     ret.pop_back();
     ret += ')';
-    if(!opt_minimize) ret += '\n';
+    if(!opt_minify) ret += '\n';
     // commands
     ret += cs.second->generate(ind+1);
     // end of case: ;;
-    if(opt_minimize && ret[ret.size()-1] == '\n') // ;; can be right after command
+    if(opt_minify && !prev_is_heredoc && ret[ret.size()-1] == '\n') // ;; can be right after command
       ret.pop_back();
-    ret += indented(";;\n", ind+1);
+    ret += indented(";;", ind+1);
+    if(!opt_minify)
+      ret+="\n";
   }
 
-  // remove ;; from last case
-  if(opt_minimize)
+  // replace ;; from last case with \n
+  if(this->cases.size()>0 && opt_minify)
   {
-    ret.erase(ret.size()-3, 2);
+    ret.pop_back();
+    ret.pop_back();
+    ret+='\n';
   }
 
   // close case
   ind--;
   ret += indented("esac", ind);
 
-  ret += generate_redirs(ind, ret);
+  ret += generate_redirs(ind, ret, ctx);
   return ret;
 }
 
-std::string cmd::generate(int ind)
+std::string cmd_t::generate(int ind, generate_context* ctx)
 {
   std::string ret;
-  // var assigns
-  for(auto it: var_assigns)
-    ret += it.first + '=' + it.second->generate(ind) + ' ';
 
-  if(args==nullptr || args->size()<=0)
+  bool has_args=false;
+
+  // pre-cmd var assigns
+  for(auto it: var_assigns)
   {
+    has_args=true;
+    if(it.first != nullptr)
+      ret += it.first->generate(ind);
+    if(it.second != nullptr)
+      ret += it.second->generate(ind);
+    ret += ' ';
+  }
+
+  // is a varassign cmd
+  if(is_cmdvar)
+  {
+    ret += args->generate(ind) + ' ';
+    for(auto it: cmd_var_assigns)
+    {
+      if(it.first != nullptr)
+        ret += it.first->generate(ind);
+      if(it.second != nullptr)
+        ret += it.second->generate(ind);
+      ret += ' ';
+    }
     ret.pop_back();
     return ret;
   }
 
-  // command
-  ret += args->generate(ind);
-  // delete potential trailing space
-  if(ret[ret.size()-1] == ' ')
-    ret.pop_back();
+  // cmd itself
+  if(args!=nullptr && args->size()>0)
+  {
+    has_args=true;
+    // command
+    ret += args->generate(ind);
+    // delete potential trailing space
+    if(ret.size()>2 && ret[ret.size()-1] == ' ' && ret[ret.size()-2] != '\\')
+      ret.pop_back();
+  }
+  else // empty command: remove trailing space
+  {
+    if(ret.size()>0)
+      ret.pop_back();
+  }
 
-  ret += generate_redirs(ind, ret);
+  std::string redirs = generate_redirs(ind, ret, ctx);
+  if(!has_args)
+    redirs.erase(redirs.begin());
+  ret += redirs;
   return ret;
 }
 
 // SUBARG
 
-std::string subshell_subarg::generate(int ind)
+std::string subarg_subshell_t::generate(int ind)
+{
+  std::string r = sbsh->generate(ind);
+  if(backtick) {
+    r[0] = '`';
+    r[r.size()-1] = '`';
+    return r;
+  }
+  else
+    return '$' + r;
+}
+
+std::string subarg_procsub_t::generate(int ind)
+{
+  if(is_output)
+    return '>' + sbsh->generate(ind);
+  else
+    return '<' + sbsh->generate(ind);
+}
+
+std::string subarg_arithmetic_t::generate(int ind)
+{
+  std::string ret;
+  ret += "$((";
+  if(!opt_minify) ret += ' ';
+  ret += arith->generate(ind);
+  if(!opt_minify) ret += ' ';
+  ret += "))";
+  return ret;
+}
+
+// ARITHMETIC
+
+std::string arithmetic_operation_t::generate(int ind)
+{
+  std::string ret;
+  if(precedence)
+  {
+    ret += oper;
+    if(!opt_minify) ret += ' ';
+    ret += val1->generate(ind);
+  }
+  else
+  {
+    ret += val1->generate(ind);
+    if(!opt_minify) ret += ' ';
+    ret += oper;
+    if(!opt_minify) ret += ' ';
+    ret += val2->generate(ind);
+  }
+  return ret;
+}
+
+std::string arithmetic_parenthesis_t::generate(int ind)
+{
+  std::string ret;
+  ret += '(';
+  if(!opt_minify) ret += ' ';
+    ret += val->generate(ind);
+  if(!opt_minify) ret += ' ';
+    ret += ')';
+  return ret;
+}
+
+std::string arithmetic_subshell_t::generate(int ind)
 {
   return '$' + sbsh->generate(ind);
 }
 
-std::string manipulation_subarg::generate(int ind)
+std::string arithmetic_variable_t::generate(int ind)
 {
-  if(size)
-    return "${#" + varname + "}";
-  else
-    return "${" + varname + manip->generate(ind) + "}";
+  std::string ret=var->generate(ind);
+  if(is_num(ret[0]) || is_in(ret[0], SPECIAL_VARS) || var->is_manip)
+    return '$' + ret;
+  return ret;
 }
 
+std::string variable_t::generate(int ind)
+{
+  std::string ret;
+  if(is_manip)
+  {
+    ret += '{';
+    if(precedence && manip!=nullptr)
+      ret += manip->generate(ind);
+  }
+  ret += varname;
+  if(index!=nullptr)
+    ret += '[' + index->generate(ind) + ']';
+  if(is_manip)
+  {
+    if(!precedence && manip!=nullptr)
+      ret += manip->generate(ind);
+    ret += '}';
+  }
+  return ret;
+}
+
+
 // TEMPLATE
 
 // std::string thing::generate(int ind)

src/main.cpp

@@ -1,45 +1,26 @@
 #include <iostream>
 
 #include <string.h>
+#include <unistd.h>
+
+#include <fstream>
 
 #include <ztd/options.hpp>
 #include <ztd/shell.hpp>
 
-#include <unistd.h>
-
 #include "util.hpp"
 #include "struc.hpp"
 #include "parse.hpp"
 #include "options.hpp"
 #include "recursive.hpp"
-#include "minimize.hpp"
+#include "minify.hpp"
 #include "resolve.hpp"
 #include "processing.hpp"
+#include "debashify.hpp"
+#include "exec.hpp"
+#include "shellcode.hpp"
 
-#include "version.h"
-#include "g_version.h"
-
-void oneshot_opt_process(const char* arg0)
-{
-  if(options['h'])
-  {
-    print_help(arg0);
-    exit(0);
-  }
-  else if(options["version"])
-  {
-    printf("%s %s%s\n", arg0, VERSION_STRING, VERSION_SUFFIX);
-    printf("%s\n", VERSION_SHA);
-    exit(0);
-  }
-  else if(options["help-commands"])
-  {
-    print_include_help();
-    printf("\n\n");
-    print_resolve_help();
-    exit(0);
-  }
-}
+#include "errcodes.h"
 
 int main(int argc, char* argv[])
 {
@@ -47,72 +28,90 @@ int main(int argc, char* argv[])
 
   int ret=0;
 
+  bool optstop=false;
+
+  shmain *sh=nullptr, *tsh=nullptr;
   try
   {
-    args=options.process(argc, argv, false, true);
-  }
-  catch(std::exception& e)
-  {
-    std::cerr << e.what() << std::endl;
-    return 1;
-  }
-
-  oneshot_opt_process(argv[0]);
-
-  // resolve input
-  std::string file;
-  if(args.size() > 0) // argument provided
-  {
-    if(args[0] == "-" || args[0] == "/dev/stdin") //stdin
+    args=options.process(argc, argv, {.stop_on_argument=true, .output_doubledash=true} );
+    if( args.size()>0 && args[0] == "--" )
     {
-      file = "/dev/stdin";
+      optstop=true;
+      args.erase(args.begin());
+    }
+
+
+    oneshot_opt_process(argv[0]);
+
+    // resolve input
+    std::string file;
+    if(args.size() > 0) // argument provided
+    {
+      if(args[0] == "-" || args[0] == "/dev/stdin") //stdin
+      {
+        file = "/dev/stdin";
+      }
+      else
+      {
+        file=args[0];
+      }
     }
     else
     {
-      file=args[0];
+      if(isatty(fileno(stdin))) // stdin is interactive
+      {
+        print_help(argv[0]);
+        return ERR_HELP;
+      }
+      else // is piped
+      {
+        file = "/dev/stdin";
+        args.push_back("/dev/stdin");
+      }
     }
-  }
-  else
-  {
-    if(isatty(fileno(stdin))) // stdin is interactive
-    {
-      print_help(argv[0]);
-      return 1;
-    }
-    else // is piped
-    {
-      file = "/dev/stdin";
-      args.push_back("/dev/stdin");
-    }
-  }
 
-  // parsing
+    // parsing
+
+    sh = new shmain;
 
-  shmain* sh = new shmain(new list);
-  shmain* tsh = nullptr;
-  try
-  {
     bool is_exec = false;
     bool first_run = true;
 
     // do parsing
+    bool shebang_is_bin=false;
+    bool parse_bash=false;
+    parse_context ctx;
+    std::string binshebang;
     for(uint32_t i=0 ; i<args.size() ; i++)
     {
       std::string file = args[i];
       std::string filecontents=import_file(file);
-      // parse
-      g_origin=file;
-      if(!add_include(file))
-        continue;
-      tsh = parse_text(filecontents, file);
-      // resolve shebang
+      std::string shebang=filecontents.substr(0,filecontents.find('\n'));
+      if(shebang.substr(0,2) != "#!")
+        shebang="#!/bin/sh";
+      // resolve shebang and parse leftover options
       if(first_run)
       {
         first_run=false;
         // resolve shebang
-        bool shebang_is_bin = ( basename(argv[0]) == basename(tsh->shebang) );
-        if(shebang_is_bin)
-          tsh->shebang="#!/bin/sh";
+        if(options["lxsh"])
+        {
+          shebang_is_bin = true;
+          parse_bash = true;
+          binshebang = basename(shebang);
+          shebang = "#!/usr/bin/env lxsh";
+        }
+        else if(options["bash"])
+        {
+          parse_bash=true;
+          shebang = "#!/usr/bin/env bash";
+        }
+        else
+        {
+          binshebang = basename(shebang);
+          shebang_is_bin = ( basename(argv[0]) == binshebang );
+          parse_bash = (options["debashify"] || binshebang == "bash" || binshebang == "lxsh");
+        }
 
         // detect if need execution
         if(options['e'])
@@ -122,46 +121,63 @@ int main(int argc, char* argv[])
         else
           is_exec = shebang_is_bin;
 
-        if(!is_exec && args.size() > 1) // not exec: parse options on args
-        {
+        if(!is_exec && args.size() > 1 && !optstop) // not exec: parse options on args
           args=options.process(args);
+
+        if(!is_exec && options['e'])
+          throw std::runtime_error("Option -e must be before file");
+
+        if(shebang_is_bin) // enable debashify option
+        {
+          shebang="#!/bin/sh";
+          options["debashify"].activated=true;
         }
 
         oneshot_opt_process(argv[0]);
         get_opts();
+
       }
+      // parse
+      if(!add_include(file))
+        continue;
 
-      /* mid processing */
-      // resolve/include
-      if(g_include || g_resolve)
-      {
-        resolve(tsh);
-      }
+      ctx.data=filecontents.data();
 
-      // concatenate to main
-      sh->concat(tsh);
-      delete tsh;
-      tsh = nullptr;
-
-      // is exec: break and exec
+      ctx = make_context(filecontents, file, parse_bash);
       if(is_exec)
-        break;
-    }
+      {
+        delete sh;
+        sh = nullptr;
+        args.erase(args.begin());
+        return exec_process(shebang.substr(2), args, ctx);
+      }
+      else
+      {
+        auto pp = parse_text(ctx);
+        tsh = pp.first;
+        if(options["bash"])
+          tsh->shebang = "#!/usr/bin/env bash";
+        ctx = pp.second;
+        if(shebang_is_bin) // resolve lxsh shebang to sh
+          tsh->shebang="#!/bin/sh";
 
-    // processing before output
-    // minimize
-    if(options['m'])
-      opt_minimize=true;
+        /* mid processing */
+        // resolve/include
+        if(g_include || g_resolve)
+          resolve(tsh, ctx);
+
+        // concatenate to main
+        sh->concat(tsh);
+        delete tsh;
+        tsh = nullptr;
+      }
+    } // end of argument parse
+
+    // pre-listing modifiers
     if(options["remove-unused"])
       delete_unused( sh, re_var_exclude, re_fct_exclude );
-    if(options["minimize-var"])
-      minimize_var( sh, re_var_exclude );
-    if(options["minimize-fct"])
-      minimize_fct( sh, re_fct_exclude );
-    // other processing
-    if(options["unset-var"])
-      add_unset_variables( sh, re_var_exclude );
 
+    // list outputs
     if(options["list-var"])
       list_vars(sh, re_var_exclude);
     else if(options["list-var-def"])
@@ -172,44 +188,101 @@ int main(int argc, char* argv[])
       list_fcts(sh, re_fct_exclude);
     else if(options["list-cmd"])
       list_cmds(sh, regex_null);
-    else if(is_exec)
+    // output
+    else
     {
-      ret = execute(sh, args);
-    }
-    else if(options['o']) // file output
-    {
-      std::string destfile=options['o'];
-      // resolve - to stdout
-      if(destfile == "-")
-        destfile = "/dev/stdout";
-      // output
-      std::ofstream(destfile) << sh->generate();
-      // don't chmod on /dev/
-      if(destfile.substr(0,5) != "/dev/")
-      ztd::exec("chmod", "+x", destfile);
-    }
-    else // to console
-    {
-      std::cout << sh->generate();
+      // post-listing modifiers
+      // implement commands
+      std::set<std::string> req_fcts;
+      if(shebang_is_bin && !options["no-extend"])
+        req_fcts = find_lxsh_commands(sh);
+      if(options["debashify"])
+        concat_sets(req_fcts, debashify(sh) );
+
+      add_lxsh_fcts(sh, req_fcts);
+
+      // processing before output
+      // minify
+      strmap_t varmap, fctmap;
+      if(options['m'])
+      {
+        opt_minify=true;
+        minify_generic(sh);
+      }
+      if(options['A']) {
+        read_minmap(options['A'].argument, &varmap, &fctmap);
+        recurse(r_replace_var, sh, &varmap);
+        recurse(r_replace_fct, sh, &fctmap);
+      }
+      else if(options["minify-var"] && options["minify-fct"]) {
+        // optimization: get everything in one go
+        allmaps_get(sh, re_var_exclude, re_fct_exclude, regex_null);
+        varmap = minify_var( sh, re_var_exclude );
+        fctmap = minify_fct( sh, re_fct_exclude );
+      }
+      else if(options["minify-var"]) {
+        varmap = minify_var( sh, re_var_exclude );
+      }
+      else if(options["minify-fct"]) {
+        fctmap = minify_fct( sh, re_fct_exclude );
+      }
+      // other processing
+      if(options["unset-var"])
+        add_unset_variables( sh, re_var_exclude );
+
+
+      if(options['P']) {
+        std::ofstream(options['P'].argument) << gen_minmap(varmap, "var") << gen_minmap(fctmap, "fct");
+      }
+
+  #ifdef DEBUG_MODE
+      if(options['J'])
+      {
+        std::cout << gen_json_struc(sh) << std::endl;
+      }
+      else
+  #endif
+
+      if(options['o']) // file output
+      {
+        std::string destfile=options['o'];
+        // resolve - to stdout
+        if(destfile == "-")
+          destfile = "/dev/stdout";
+        // output
+        std::ofstream(destfile) << sh->generate(g_shebang, 0);
+        // don't chmod on /dev/
+        if(destfile.substr(0,5) != "/dev/")
+          ztd::exec("chmod", "+x", destfile);
+      }
+      else // to console
+      {
+        std::cout << sh->generate(g_shebang, 0);
+      }
     }
   }
-  catch(ztd::format_error& e)
+  catch(format_error& e)
   {
     if(tsh != nullptr)
       delete tsh;
     delete sh;
     printFormatError(e);
-    return 100;
+    return ERR_PARSE;
+  }
+  catch(ztd::option_error& e)
+  {
+    std::cerr << e.what() << std::endl;
+    return ERR_OPT;
   }
   catch(std::runtime_error& e)
   {
     if(tsh != nullptr)
       delete tsh;
-    delete sh;
+    if(sh != nullptr)
+      delete sh;
     std::cerr << e.what() << std::endl;
-    return 2;
+    return ERR_RUNTIME;
   }
-
   delete sh;
 
   return ret;

src/minify.cpp

@@ -0,0 +1,727 @@
+#include "minify.hpp"
+
+#include <fstream>
+
+#include "parse.hpp"
+#include "recursive.hpp"
+#include "processing.hpp"
+#include "util.hpp"
+
+std::vector<subarg_t*> cmd_t::subarg_vars()
+{
+  std::vector<subarg_t*> ret;
+  if(args==nullptr || args->size()<=0)
+    return ret;
+
+  if(this->is_argvar())
+  {
+    for(uint32_t i=1; i<args->size(); i++)
+    {
+      arg_t* ta = args->args[i];
+      if(ta->sa.size() < 1 || ta->sa[0]->type != _obj::subarg_string)
+        continue;
+      if(ta->sa.size() >= 1 && is_varname(ta->sa[0]->generate(0)))
+        ret.push_back(ta->sa[0]);
+    }
+  }
+
+  return ret;
+}
+
+/** RECURSIVES **/
+
+bool r_replace_fct(_obj* in, strmap_t* fctmap)
+{
+  switch(in->type)
+  {
+    case _obj::block_function: {
+      function_t* t = dynamic_cast<function_t*>(in);
+      auto el=fctmap->find(t->name);
+      if(el!=fctmap->end())
+        t->name = el->second;
+    }; break;
+    case _obj::block_cmd: {
+      cmd_t* t = dynamic_cast<cmd_t*>(in);
+      std::string cmdname = t->arg_string(0);
+      auto el=fctmap->find(cmdname);
+      if(el!=fctmap->end())
+      {
+        delete t->args->args[0];
+        t->args->args[0] = new arg_t(el->second);
+      }
+    }; break;
+    default: break;
+  }
+  return true;
+}
+
+bool r_replace_var(_obj* in, strmap_t* varmap)
+{
+  switch(in->type)
+  {
+    case _obj::variable: {
+      variable_t* t = dynamic_cast<variable_t*>(in);
+      auto el=varmap->find(t->varname);
+      if(el!=varmap->end())
+        t->varname = el->second;
+    }; break;
+    default: break;
+  }
+  return true;
+}
+
+const char* singlequote_escape_char=" \\\t!\"()|&*?~><#$";
+const char* doublequote_escape_char="  \t'|&\\*()?~><#$";
+uint32_t count_escape_char(std::string& in, uint32_t i, bool doublequote, std::string** estr, uint32_t* ei) {
+  if( (  doublequote && is_in(in[i], doublequote_escape_char) ) ||
+      ( !doublequote && is_in(in[i], singlequote_escape_char) ) ) {
+    *estr = &in;
+    *ei = i;
+    return 1;
+  }
+  else if(in[i] == '\n') // \n: can't remove quotes
+    return 2;
+  return 0;
+}
+
+uint32_t count_escape_chars(std::string const& in, bool doublequote)
+{
+  uint32_t r=0;
+  for(uint32_t i=0; i<in.size(); i++)
+  {
+    if(doublequote && is_in(in[i], doublequote_escape_char))
+      r++;
+    else if(!doublequote && is_in(in[i], singlequote_escape_char))
+      r++;
+    else if(in[i] == '\n') // \n: can't remove quotes
+      return 2;
+    else if(in[i] == '$')
+    {
+      if(i+1>=in.size())
+        continue;
+      else if(is_in(in[i+1], SPECIAL_VARS) || is_alphanum(in[i+1]) || in[i+1] == '_' || in[i+1] == '(')
+      {
+        if(doublequote) // doublequote: can't remove otherwise not quoted var
+          return 2;
+        r++;
+      }
+    }
+  }
+  return r;
+}
+
+bool is_this_quote(char c, bool is_doublequote)
+{
+  if(is_doublequote)
+    return c == '"';
+  else
+    return c == '\'';
+}
+
+bool is_varname(const char c) {
+  return is_alphanum(c) || c == '_';
+}
+
+void do_minify_quotes(arg_t* in)
+{
+  auto t = in->sa.begin();
+  // global loop
+  while(true)
+  {
+    uint32_t i=0;
+    // one iteration loop
+    while(true)
+    {
+      bool doublequote=false;
+      bool prev_is_var=false;
+      bool end_is_var=false;
+      bool has_substitution=false;
+      std::string* strstart = nullptr;
+      uint32_t quotestart=0;
+      std::string* strend = nullptr;
+      uint32_t quoteend=0;
+      std::string* escapestr = nullptr;
+      uint32_t escapepos=0;
+      uint32_t ce=0;
+      // loop to find start of quote
+      while(true)
+      {
+        // reached end: quit
+        if(t == in->sa.end())
+          return;
+        while((*t)->type != _obj::subarg_string)
+        {
+          // previous is alphanum var: removing quote can change varname
+          if((*t)->type == _obj::subarg_variable) {
+            subarg_variable_t* vs = dynamic_cast<subarg_variable_t*>(*t);
+            if(vs->var != nullptr && !vs->var->is_manip && vs->var->varname.size()>0 && !(is_in(vs->var->varname[0], SPECIAL_VARS) || is_num(vs->var->varname[0]) ) )
+              prev_is_var = true;
+          }
+          else
+            prev_is_var = false;
+          t++;
+          // quit when reached end of arg
+          if(t == in->sa.end())
+            return;
+          i=0;
+        }
+        std::string& val = dynamic_cast<subarg_string_t*>(*t)->val;
+        // don't attempt if <= 2 chars
+        if(in->sa.size() == 1 && val.size() <= 2)
+          return;
+        while(i<val.size() && !( val[i] == '\'' || val[i] == '"') )
+        {
+          if(val[i] == '\\')
+            i++;
+          i++;
+        }
+        // if found: break and go to next step
+        if(i<val.size()) {
+          if(val[i] == '"')
+            doublequote=true;
+          strstart=&val;
+          quotestart=i;
+          i++;
+          break;
+        }
+        else {
+          t++;
+          i=0;
+        }
+      } // end of quote start loop
+      // loop to end of quote
+      while(true)
+      {
+        // reached end: quit
+        if(t == in->sa.end())
+          return;
+        while((*t)->type != _obj::subarg_string)
+        {
+          // previous is alphanum var: removing quote can change varname
+          if((*t)->type == _obj::subarg_variable) {
+            subarg_variable_t* vs = dynamic_cast<subarg_variable_t*>(*t);
+            if(vs->var != nullptr && !vs->var->is_manip && vs->var->varname.size()>0 && !(is_in(vs->var->varname[0], SPECIAL_VARS) || is_num(vs->var->varname[0]) ) )
+              end_is_var = true;
+          }
+          else
+            end_is_var = false;
+          has_substitution=true;
+          t++;
+          // quit when reached end of arg
+          if(t == in->sa.end())
+            return;
+          i=0;
+        }
+        std::string& val = dynamic_cast<subarg_string_t*>(*t)->val;
+        if(doublequote)
+        {
+          while(i<val.size() && val[i] != '"')
+          {
+            if(val[i] == '\\') {
+              ce += count_escape_char(val, i++, doublequote, &escapestr, &escapepos);
+            }
+            ce += count_escape_char(val, i++, doublequote, &escapestr, &escapepos);
+          }
+          if(i>=val.size()) { // end before finding quote: continue looping
+            t++;
+            i=0;
+            continue;
+          }
+        }
+        else
+        {
+          while(i<val.size() && val[i] != '\'')
+            ce += count_escape_char(val, i++, doublequote, &escapestr, &escapepos);
+          if(i>=val.size()) { // end before finding quote: continue looping
+            t++;
+            i=0;
+            continue;
+          }
+        }
+        strend=&val;
+        quoteend=i;
+        break;
+      } // end of quote end loop
+      // has a substitution that can expand: don't dequote
+      if(!in->forcequoted && has_substitution) {
+        i++;
+        continue;
+      }
+      // too many escapes: don't dequote
+      if(ce > 1) {
+        i++;
+        continue;
+      }
+      // removing quotes changes variable name: don't dequote
+      if( ( prev_is_var && quotestart == 0 && strstart->size()>1 && is_varname((*strstart)[1]) ) ||
+          ( end_is_var && quoteend == 0 && strend->size()>1 && is_varname((*strend)[1])) ) {
+        i++;
+        continue;
+      }
+
+      // prev char is a $ would create variable names: don't dequote
+      if( quotestart >= 1 && (*strstart)[quotestart-1] == '$' && (!doublequote ||
+            ( strstart->size()>2 && is_varname((*strstart)[quotestart+1])))
+          ) {
+        i++;
+        continue;
+      }
+
+      // do dequote
+      strend->erase(quoteend, 1);
+      // needs one escape
+      if(ce == 1) {
+        escapestr->insert(escapepos, "\\");
+      }
+      strstart->erase(quotestart, 1);
+
+    }
+  }
+
+}
+
+void do_minify_dollar(subarg_string_t* in)
+{
+  std::string& val = in->val;
+  for(uint32_t i=0; i<val.size(); i++) {
+    // skip singlequote strings
+    if(val[i] == '\'') {
+      i++;
+      while(val[i] != '\'')
+        i++;
+    }
+    // has \$
+    if(i+1<val.size() && val[i] == '\\' && val[i+1] == '$') {
+      // char after $ is a varname
+      if(i+2<val.size() && (is_varname(val[i+2]) || is_in(val[i+2], SPECIAL_VARS) || val[i+2] == '{') )
+        continue;
+      val.erase(i, 1);
+    }
+  }
+}
+
+bool r_minify_useless_quotes(_obj* in)
+{
+  switch(in->type)
+  {
+    case _obj::arg: {
+      arg_t* t = dynamic_cast<arg_t*>(in);
+      do_minify_quotes(t);
+    }; break;
+    case _obj::subarg_string: {
+      subarg_string_t* t = dynamic_cast<subarg_string_t*>(in);
+      do_minify_dollar(t);
+    }; break;
+    case _obj::redirect: {
+      // for redirects: don't minify quotes on here documents
+      redirect_t* t = dynamic_cast<redirect_t*>(in);
+      if(t->here_document != nullptr)
+      {
+        recurse(r_minify_useless_quotes, t->target);
+        for(auto it: t->here_document->sa)
+        {
+          if(it->type!=_obj::subarg_string) {
+            recurse(r_minify_useless_quotes, it);
+          }
+        }
+        // don't recurse on the rest
+        return false;
+      }
+    } break;
+    default: break;
+  }
+  return true;
+}
+
+/** NAME MINIFYING **/
+
+char nchar(uint32_t n)
+{
+  if(n<26)
+    return 'a'+n;
+  else if(n<52)
+    return 'A'+(n-26);
+  else if(n==52)
+    return '_';
+  else if(n<63)
+    return '0'+(n-53);
+  else
+    return 0;
+}
+
+std::string minimal_name(uint32_t n)
+{
+  if(n<53)
+  {
+    std::string ret;
+    ret += nchar(n);
+    return ret;
+  }
+  else
+  {
+    uint32_t k=n%53;
+    uint32_t q=n/53;
+    std::string ret;
+    ret += nchar(k);
+    ret += nchar(q);
+    while(q>64)
+    {
+      q /= 64;
+      ret += nchar(q);
+    }
+    return ret;
+  }
+}
+
+// vars: input variables
+// excluded: excluded variables to make sure there is no collision
+strmap_t gen_minimal_map(countmap_t const& vars, set_t const& excluded)
+{
+  strmap_t ret;
+  auto ordered = sort_by_value(vars);
+  uint32_t n=0;
+  for(std::pair<std::string,uint32_t> it: ordered)
+  {
+    std::string newname;
+    do {
+      newname = minimal_name(n);
+      n++;
+    } while( excluded.find(newname) != excluded.end() );
+    ret.insert(std::make_pair(it.first, newname));
+  }
+  return ret;
+}
+
+// calls
+
+strmap_t minify_var(_obj* in, std::regex const& exclude)
+{
+  // countmap_t vars;
+  set_t excluded;
+  strmap_t varmap;
+  // get vars
+  varmap_get(in, exclude);
+  // concatenate excluded and reserved
+  concat_sets(excluded, m_excluded_var);
+  concat_sets(excluded, all_reserved_words);
+  // create mapping
+  varmap=gen_minimal_map(m_vars, excluded);
+  // perform replace
+  recurse(r_replace_var, in, &varmap);
+  require_rescan_var();
+  return varmap;
+}
+
+strmap_t minify_fct(_obj* in, std::regex const& exclude)
+{
+  // countmap_t fcts, cmdmap;
+  set_t excluded, unsets;
+  strmap_t fctmap;
+  // get fcts and cmds
+  fctcmdmap_get(in, exclude, regex_null);
+  recurse(r_get_unsets, in, &unsets);
+  // concatenate cmds, excluded and reserved
+  excluded=map_to_set(m_cmds);
+  exclude_sets(excluded, map_to_set(m_fcts));
+  concat_sets(excluded, m_excluded_fct);
+  concat_sets(excluded, unsets);
+  concat_sets(excluded, all_reserved_words);
+  // create mapping
+  m_fcts = combine_common(m_fcts, m_cmds);
+  fctmap=gen_minimal_map(m_fcts, excluded);
+  // perform replace
+  recurse(r_replace_fct, in, &fctmap);
+  require_rescan_fct();
+  require_rescan_cmd();
+  return fctmap;
+}
+
+bool delete_unused_fct(_obj* in, std::regex const& exclude)
+{
+  set_t unused;
+  // get fcts and cmds
+  fctcmdmap_get(in, exclude, regex_null);
+  // find unused fcts
+  for(auto it: m_fcts)
+  {
+    if(m_cmds.find(it.first) == m_cmds.end())
+      unused.insert(it.first);
+  }
+  // perform deletion
+  if(unused.size()>0)
+  {
+    recurse(r_delete_fct, in, &unused);
+    require_rescan_all();
+    return true;
+  }
+  else
+    return false;
+}
+
+bool delete_unused_var(_obj* in, std::regex const& exclude)
+{
+  set_t unused;
+  // get fcts and cmds
+  varmap_get(in, exclude);
+  // find unused vars
+  for(auto it: m_vardefs)
+  {
+    if(it.first!="" && m_varcalls.find(it.first) == m_varcalls.end())
+      unused.insert(it.first);
+  }
+  // perform deletion
+  if(unused.size()>0)
+  {
+    recurse(r_delete_var, in, &unused);
+    require_rescan_all();
+    return true;
+  }
+  else
+    return false;
+}
+
+bool delete_unused_both(_obj* in, std::regex const& var_exclude, std::regex const& fct_exclude)
+{
+  set_t unused_var, unused_fct;
+  // get all
+  allmaps_get(in, var_exclude, fct_exclude, regex_null);
+  // find unused
+  for(auto it: m_vardefs)
+  {
+    if(it.first!="" && m_varcalls.find(it.first) == m_varcalls.end())
+      unused_var.insert(it.first);
+  }
+  for(auto it: m_fcts)
+  {
+    if(m_cmds.find(it.first) == m_cmds.end())
+      unused_fct.insert(it.first);
+  }
+  if(unused_var.size()>0 || unused_fct.size()>0)
+  {
+    recurse(r_delete_varfct, in, &unused_var, &unused_fct);
+    require_rescan_all();
+    return true;
+  }
+  return false;
+}
+
+void delete_unused(_obj* in, std::regex const& var_exclude, std::regex const& fct_exclude)
+{
+  while(delete_unused_both(in, var_exclude, fct_exclude));
+  // keep deleting until both no deletion
+}
+
+
+// minify ${var} to $var
+bool r_minify_empty_manip(_obj* in)
+{
+  switch(in->type)
+  {
+    case _obj::arg: {
+      arg_t* t = dynamic_cast<arg_t*>(in);
+      for(uint32_t i=0; i<t->sa.size(); i++)
+      {
+        if(t->sa[i]->type == _obj::subarg_variable)
+        {
+          // has to be a variable
+          subarg_variable_t* ss = dynamic_cast<subarg_variable_t*>(t->sa[i]);
+          if(ss->var->is_manip)
+          {
+            // if is a manip: possibility to skip it
+            if(ss->var->index != nullptr) // is a var bash array: skip
+              return true;
+            if(i+1<t->sa.size() && t->sa[i+1]->type == _obj::subarg_string)
+            {
+              // if next subarg is a string: check its first char
+              subarg_string_t* ss = dynamic_cast<subarg_string_t*>(t->sa[i+1]);
+              char c = ss->val[0];
+              // if its first would extend the var name: skip
+              if(is_alphanum(c) || c == '_')
+                continue;
+            }
+            // if has no actual manipulation operation: set it to not manip
+            if(ss->var->manip == nullptr || ss->var->manip->sa.size() == 0)
+              ss->var->is_manip = false;
+          }
+        }
+      }
+    }; break;
+    default: break;
+  }
+  return true;
+}
+
+pipeline_t* do_one_minify_single_block(block_t* in)
+{
+  pipeline_t* ret=nullptr;
+  list_t* l=nullptr;
+  if(in->type == _obj::block_brace)
+    l = dynamic_cast<brace_t*>(in)->lst;
+  else if(in->type == _obj::block_subshell)
+    l = dynamic_cast<subshell_t*>(in)->lst;
+
+  if(l == nullptr)
+    return nullptr;
+
+  // not a single pipeline: not applicable
+  if(l->cls.size() != 1 || l->cls[0]->pls.size() != 1)
+    return nullptr;
+
+  ret = l->cls[0]->pls[0];
+
+  // if is a subshell and has some env set: don't remove it
+  if(in->type == _obj::block_subshell && has_env_set(ret))
+    return nullptr;
+
+  // has a non-stdout/stdin redirect: not applicable
+  for(auto it: in->redirs) {
+    if(!is_in(it->op[0], "<>") )
+      return nullptr;
+  }
+
+  return ret;
+}
+
+bool r_minify_single_block(_obj* in)
+{
+  switch(in->type)
+  {
+    case _obj::pipeline: {
+      bool has_operated=false;
+      do
+      {
+        // loop operating on current
+        // (if has operated, current object has changed)
+        has_operated=false;
+        pipeline_t* t = dynamic_cast<pipeline_t*>(in);
+        for(uint32_t i=0; i<t->cmds.size(); i++)
+        {
+          pipeline_t* ret = do_one_minify_single_block(t->cmds[i]);
+          if(ret != nullptr) {
+            // concatenate redirects
+            block_t* firstb = ret->cmds[0];
+            block_t* lastb = ret->cmds[ret->cmds.size()-1];
+            uint32_t j1=0, j2=0;
+            for(uint32_t j=0; j<t->cmds[i]->redirs.size(); j++) {
+              if(t->cmds[i]->redirs[j]->op[0] == '<') {
+                firstb->redirs.insert(firstb->redirs.begin()+j1, t->cmds[i]->redirs[j]);
+                j1++;
+              }
+              else {
+                lastb->redirs.insert(lastb->redirs.begin()+j2, t->cmds[i]->redirs[j]);
+                j2++;
+              }
+            }
+
+            // deindex
+            t->cmds[i]->redirs.resize(0);
+            if(t->cmds[i]->type == _obj::block_brace)
+              dynamic_cast<brace_t*>(t->cmds[i])->lst->cls[0]->pls[0] = nullptr;
+            else if(t->cmds[i]->type == _obj::block_subshell)
+              dynamic_cast<subshell_t*>(t->cmds[i])->lst->cls[0]->pls[0] = nullptr;
+
+            // replace value
+            delete t->cmds[i];
+            t->cmds.erase(t->cmds.begin()+i);
+            for(auto it: ret->cmds) {
+              t->cmds.insert(t->cmds.begin()+i, it);
+              i++;
+            }
+
+            has_operated=true;
+          }
+        }
+      }
+      while(has_operated);
+    }; break;
+    default: break;
+  }
+  return true;
+}
+
+bool r_has_backtick(_obj* in, bool* r)
+{
+  if(*r)
+    return false;
+  switch(in->type)
+  {
+    case _obj::subarg_subshell: {
+      subarg_subshell_t* t = dynamic_cast<subarg_subshell_t*>(in);
+      if(t->backtick) {
+        *r = true;
+        return false;
+      }
+    }; break;
+    case _obj::subarg_string: {
+      subarg_string_t* t = dynamic_cast<subarg_string_t*>(in);
+      if(t->val.find('\\') != std::string::npos)
+        *r = true;
+    }; break;
+    default: break;
+  }
+  return true;
+}
+
+bool r_minify_backtick(_obj* in)
+{
+  switch(in->type)
+  {
+    case _obj::subarg_subshell: {
+      subarg_subshell_t* t = dynamic_cast<subarg_subshell_t*>(in);
+      if(!t->backtick) {
+        bool has_backtick_child=false;
+        recurse(r_has_backtick, t->sbsh, &has_backtick_child);
+        if(has_backtick_child)
+          return false;
+        t->backtick = true;
+      }
+      return false;
+    }; break;
+    default: break;
+  }
+  return true;
+}
+
+// optimisation for processors that don't have recurse-cancellation
+bool r_minify(_obj* in)
+{
+  r_minify_empty_manip(in);
+  r_minify_single_block(in);
+  r_do_string_processor(in);
+  return true;
+}
+
+void minify_generic(_obj* in)
+{
+  recurse(r_minify, in);
+  recurse(r_minify_backtick, in);
+  recurse(r_minify_useless_quotes, in);
+}
+
+std::string gen_minmap(strmap_t const& map, std::string const& prefix)
+{
+  std::string ret;
+  for(auto it: map) {
+    ret += strf("%s %s %s\n", prefix.c_str(), it.second.c_str(), it.first.c_str());
+  }
+  return ret;
+}
+
+void read_minmap(std::string const& filepath, strmap_t* varmap, strmap_t* fctmap)
+{
+  std::ifstream file(filepath);
+  std::string ln;
+  while(std::getline(file, ln)) {
+    size_t s1, s2, s3;
+    s1 = ln.find(' ');
+    s2 = ln.find(' ', s1+1);
+    s3 = ln.find(' ', s2+1);
+    std::string type = ln.substr(0, s1);
+    std::string from = ln.substr(s1+1, s2-s1-1);
+    std::string to   = ln.substr(s2+1, s3-s2-1);
+    if(type == "var")
+      varmap->insert(std::make_pair(from, to));
+    else if(type == "fct")
+      fctmap->insert(std::make_pair(from, to));
+  }
+}

src/minimize.cpp

@@ -1,249 +0,0 @@
-#include "minimize.hpp"
-
-#include "recursive.hpp"
-#include "processing.hpp"
-#include "util.hpp"
-
-std::vector<subarg*> cmd::subarg_vars()
-{
-  std::vector<subarg*> ret;
-  if(args==nullptr || args->size()<=0)
-    return ret;
-
-  if(this->is_argvar())
-  {
-    for(uint32_t i=1; i<args->size(); i++)
-    {
-      arg* ta = args->args[i];
-      if(ta->sa.size() < 1 || ta->sa[0]->type != _obj::subarg_string)
-        continue;
-      if(ta->sa.size() >= 1 && is_varname(ta->sa[0]->generate(0)))
-        ret.push_back(ta->sa[0]);
-    }
-  }
-
-  return ret;
-}
-
-/** RECURSIVES **/
-
-bool r_replace_fct(_obj* in, strmap_t* fctmap)
-{
-  switch(in->type)
-  {
-    case _obj::block_function: {
-      function* t = dynamic_cast<function*>(in);
-      auto el=fctmap->find(t->name);
-      if(el!=fctmap->end())
-        t->name = el->second;
-    }; break;
-    case _obj::block_cmd: {
-      cmd* t = dynamic_cast<cmd*>(in);
-      std::string cmdname = t->firstarg_string();
-      auto el=fctmap->find(cmdname);
-      if(el!=fctmap->end())
-      {
-        delete t->args->args[0];
-        t->args->args[0] = new arg(el->second);
-      }
-    }; break;
-    default: break;
-  }
-  return true;
-}
-
-bool r_replace_var(_obj* in, strmap_t* varmap)
-{
-  switch(in->type)
-  {
-    case _obj::subarg_variable: {
-      variable_subarg* t = dynamic_cast<variable_subarg*>(in);
-      auto el=varmap->find(t->varname);
-      if(el!=varmap->end())
-        t->varname = el->second;
-    }; break;
-    case _obj::subarg_manipulation: {
-      manipulation_subarg* t = dynamic_cast<manipulation_subarg*>(in);
-      auto el=varmap->find(t->varname);
-      if(el!=varmap->end())
-        t->varname = el->second;
-    }; break;
-    case _obj::block_for: {
-      for_block* t = dynamic_cast<for_block*>(in);
-      auto it=varmap->find(t->varname);
-      if(it!=varmap->end())
-        t->varname = it->second;
-    }; break;
-    case _obj::block_cmd: {
-      cmd* t = dynamic_cast<cmd*>(in);
-      for(auto it=t->var_assigns.begin() ; it!=t->var_assigns.end() ; it++)
-      {
-        auto el=varmap->find(it->first);
-        if(el!=varmap->end())
-          it->first = el->second;
-      }
-      for(auto it: t->subarg_vars())
-      {
-        string_subarg* t = dynamic_cast<string_subarg*>(it);
-        auto el=varmap->find(get_varname(t->val));
-        if(el!=varmap->end())
-        {
-          size_t tpos=t->val.find('=');
-          if(tpos == std::string::npos)
-            t->val = el->second;
-          else
-            t->val = el->second + t->val.substr(tpos);
-        }
-      }
-    }; break;
-    default: break;
-  }
-  return true;
-}
-
-/** NAME MINIMIZING **/
-
-char nchar(uint32_t n)
-{
-  if(n<26)
-    return 'a'+n;
-  else if(n<52)
-    return 'A'+(n-26);
-  else if(n==52)
-    return '_';
-  else if(n<63)
-    return '0'+(n-53);
-  else
-    return 0;
-}
-
-std::string minimal_name(uint32_t n)
-{
-  if(n<53)
-  {
-    std::string ret;
-    ret += nchar(n);
-    return ret;
-  }
-  else
-  {
-    uint32_t k=n%53;
-    uint32_t q=n/53;
-    std::string ret;
-    ret += nchar(k);
-    ret += nchar(q);
-    while(q>64)
-    {
-      q /= 64;
-      ret += nchar(q);
-    }
-    return ret;
-  }
-}
-
-// vars: input variables
-// excluded: excluded variables to make sure there is no collision
-strmap_t gen_minimal_map(countmap_t const& vars, set_t const& excluded)
-{
-  strmap_t ret;
-  auto ordered = sort_by_value(vars);
-  uint32_t n=0;
-  for(std::pair<std::string,uint32_t> it: ordered)
-  {
-    std::string newname;
-    do {
-      newname = minimal_name(n);
-      n++;
-    } while( excluded.find(newname) != excluded.end() );
-    ret.insert(std::make_pair(it.first, newname));
-  }
-  return ret;
-}
-
-// calls
-
-void minimize_var(_obj* in, std::regex const& exclude)
-{
-  // countmap_t vars;
-  set_t excluded;
-  strmap_t varmap;
-  // get vars
-  varmap_get(in, exclude);
-  // create mapping
-  varmap=gen_minimal_map(m_vars, m_excluded_var);
-  // perform replace
-  recurse(r_replace_var, in, &varmap);
-  require_rescan_var();
-}
-
-void minimize_fct(_obj* in, std::regex const& exclude)
-{
-  // countmap_t fcts, cmdmap;
-  set_t allcmds, excluded, unsets;
-  strmap_t fctmap;
-  // get fcts and cmds
-  fctmap_get(in, exclude);
-  cmdmap_get(in, regex_null);
-  recurse(r_get_unsets, in, &unsets);
-  // concatenate cmds and excluded commands
-  allcmds=map_to_set(m_cmds);
-  concat_sets(allcmds, m_excluded_fct);
-  concat_sets(allcmds, unsets);
-  // create mapping
-  fctmap=gen_minimal_map(m_fcts, allcmds);
-  // perform replace
-  recurse(r_replace_fct, in, &fctmap);
-  require_rescan_fct();
-  require_rescan_cmd();
-}
-
-bool delete_unused_fct(_obj* in, std::regex const& exclude)
-{
-  set_t unused;
-  // get fcts and cmds
-  fctmap_get(in, exclude);
-  cmdmap_get(in, regex_null);
-  // find unused fcts
-  for(auto it: m_fcts)
-  {
-    if(m_cmds.find(it.first) == m_cmds.end())
-      unused.insert(it.first);
-  }
-  // perform deletion
-  if(unused.size()>0)
-  {
-    recurse(r_delete_fct, in, &unused);
-    require_rescan_all();
-    return true;
-  }
-  else
-    return false;
-}
-
-bool delete_unused_var(_obj* in, std::regex const& exclude)
-{
-  set_t unused;
-  // get fcts and cmds
-  varmap_get(in, exclude);
-  // find unused vars
-  for(auto it: m_vardefs)
-  {
-    if(m_varcalls.find(it.first) == m_varcalls.end())
-      unused.insert(it.first);
-  }
-  // perform deletion
-  if(unused.size()>0)
-  {
-    recurse(r_delete_var, in, &unused);
-    require_rescan_all();
-    return true;
-  }
-  else
-    return false;
-}
-
-void delete_unused(_obj* in, std::regex const& var_exclude, std::regex const& fct_exclude)
-{
-  while(delete_unused_fct(in, fct_exclude) || delete_unused_var(in, var_exclude));
-  // keep deleting until both no function and no variables were deleted
-}

src/options.cpp

@@ -1,88 +1,116 @@
 #include "options.hpp"
 
 #include "processing.hpp"
+#include "shellcode.hpp"
 
-ztd::option_set options = gen_options();
-bool opt_minimize=false;
+#include "errcodes.h"
+#include "version.h"
+#include "g_version.h"
+
+bool opt_minify=false;
+
+ztd::option_set options( {
+  ztd::option("\r  [Help]"),
+  ztd::option('h', "help",          false, "Display this help message"),
+  ztd::option("version",            false, "Display version"),
+  ztd::option("help-link-commands", false, "Print help for linker commands"),
+  ztd::option("help-extend-fcts",   false, "Print help for lxsh extension functions"),
+  ztd::option("\r  [Output]"),
+  ztd::option('o', "output",        true , "Output result script to file", "file"),
+  ztd::option('c', "stdout",        false, "Output result script to stdout"),
+  ztd::option('e', "exec",          false, "Directly execute script"),
+  ztd::option("no-shebang",         false, "Don't output shebang"),
+  ztd::option('P', "map",           true , "Output var and fct minify map to given file", "file"),
+#ifdef DEBUG_MODE
+  ztd::option("\r  [Debugging]"),
+  ztd::option('J', "json",          false, "Output the json structure"),
+#endif
+  ztd::option("\r  [Processing]"),
+  ztd::option('m', "minify",        false, "Minify code without changing functionality"),
+  ztd::option('M', "minify-full",   false, "Enable all minifying features: -m --minify-var --minify-fct --remove-unused"),
+  ztd::option('A', "apply-map",     true , "Apply var/fct minify map from given file", "file"),
+  ztd::option('C', "no-cd",         false, "Don't cd when doing %include and %resolve"),
+  ztd::option('I', "no-include",    false, "Don't resolve %include commands"),
+  ztd::option('R', "no-resolve",    false, "Don't resolve %resolve commands"),
+  ztd::option("no-extend",          false, "Don't add lxsh extension functions"),
+  ztd::option("bash",               false, "Force bash parsing"),
+  ztd::option("lxsh",               false, "Force lxsh parsing"),
+  ztd::option("debashify",          false, "Attempt to turn a bash-specific script into a POSIX shell script"),
+  ztd::option("remove-unused",      false, "Remove unused functions and variables"),
+  ztd::option("list-cmd",           false, "List all commands invoked in the script"),
+  ztd::option("\r  [Variable processing]"),
+  ztd::option("exclude-var",        true,  "List of matching regex to ignore for variable processing, separated by spaces", "list"),
+  ztd::option("no-exclude-reserved",false, "Don't exclude reserved variables"),
+  ztd::option("minify-var",         false, "Minify variable names"),
+  ztd::option("list-var",           false, "List all variables set and invoked in the script"),
+  ztd::option("list-var-def",       false, "List all variables set in the script"),
+  ztd::option("list-var-call",      false, "List all variables invoked in the script"),
+  ztd::option("unset-var",          false, "Add 'unset' to all variables at the start of the script to avoid environment interference"),
+  ztd::option("\r  [Function processing]"),
+  ztd::option("exclude-fct",        true,  "List of matching regex to ignore for function processing, separated by spaces", "list"),
+  ztd::option("minify-fct",         false, "Minify function names"),
+  ztd::option("list-fct",           false, "List all functions defined in the script")
+} );
 
 bool g_cd=false;
 bool g_include=true;
 bool g_resolve=true;
-
-ztd::option_set gen_options()
-{
-  ztd::option_set ret;
-  ret.add(
-      ztd::option("\r  [Help]"),
-      ztd::option('h', "help",          false, "Display this help message"),
-      ztd::option("version",            false, "Display version"),
-      ztd::option("help-commands",      false, "Print help for linker commands"),
-      ztd::option("\r  [Output]"),
-      ztd::option('o', "output",        true , "Output result script to file", "file"),
-      ztd::option('c', "stdout",        false, "Output result script to stdout"),
-      ztd::option('e', "exec",          false, "Directly execute script"),
-      ztd::option("\r  [Processing]"),
-      ztd::option('C', "no-cd",         false, "Don't cd when doing %include and %resolve"),
-      ztd::option('m', "minimize",      false, "Minimize code without changing functionality"),
-      ztd::option('I', "no-include",    false, "Don't resolve %include commands"),
-      ztd::option('R', "no-resolve",    false, "Don't resolve %resolve commands"),
-      ztd::option("\r  [var/fct processing]"),
-      ztd::option("minimize-var",       false, "Minimize variable names"),
-      ztd::option("minimize-fct",       false, "Minimize function names"),
-      ztd::option("exclude-var",        true,  "List of matching regex to ignore for variable processing", "list"),
-      ztd::option("exclude-fct",        true,  "List of matching regex to ignore for function processing", "list"),
-      ztd::option("no-exclude-reserved",false, "Don't exclude reserved variables"),
-      ztd::option("list-var",           false, "List all variables set and invoked in the script"),
-      ztd::option("list-var-def",       false, "List all variables set in the script"),
-      ztd::option("list-var-call",      false, "List all variables invoked in the script"),
-      ztd::option("list-fct",           false, "List all functions defined in the script"),
-      ztd::option("list-cmd",           false, "List all commands invoked in the script"),
-      ztd::option("remove-unused",      false, "Remove unused functions and variables"),
-      ztd::option("unset-var",          false, "Add 'unset' to all vars at the start of the script to avoid environment interference")
-  );
-  return ret;
-}
+bool g_shebang=true;
 
 void get_opts()
 {
   g_cd=!options['C'].activated;
   g_include=!options["no-include"].activated;
   g_resolve=!options["no-resolve"].activated;
+  g_shebang=!options["no-shebang"].activated;
   if(options["exclude-var"])
     re_var_exclude=var_exclude_regex(options["exclude-var"], !options["no-exclude-reserved"]);
   else
     re_var_exclude=var_exclude_regex("", !options["no-exclude-reserved"]);
   if(options["exclude-fct"])
     re_fct_exclude=fct_exclude_regex(options["exclude-fct"]);
+  if(options['M'])
+  {
+    options['m'].activated=true;
+    options["minify-var"].activated=true;
+    options["minify-fct"].activated=true;
+    options["remove-unused"].activated=true;
+  }
+  if(options['o'].argument == "-")
+    options['o'].argument = "/dev/stdout";
+  if(options['P'].argument == "-")
+    options['P'].argument = "/dev/stdout";
+  if(options['A'].argument == "-")
+    options['A'].argument = "/dev/stdin";
+  if(
+      options['A'] && ( options['P'] || options["minify-var"] || options["minify-fct"] )
+    ) {
+      printf("Incompatible options\n");
+      exit(ERR_OPT);
+  }
 }
 
 ztd::option_set create_include_opts()
 {
-  ztd::option_set opts;
-  opts.add(
-    ztd::option('e', false, "Escape double quotes"),
+  return std::vector<ztd::option>({
     ztd::option('C', false, "Don't cd to folder the file is in"),
     ztd::option('f', false, "Force include even if already included. Don't count as included")
-  );
-  return opts;
+  });
 }
 
 ztd::option_set create_resolve_opts()
 {
-  ztd::option_set opts;
-  opts.add(
+  return std::vector<ztd::option>({
     ztd::option('C', false, "Don't cd to folder this file is in"),
     ztd::option('f', false, "Ignore non-zero return values")
-  );
-  return opts;
+  });
 }
 
 void print_help(const char* arg0)
 {
   printf("%s [options] <file> [arg...]\n", arg0);
-  printf("Link extended shell\n");
-  printf("Include files and resolve commands on build time\n");
-  printf("See --help-commands for help on linker commands\n");
+  printf("Extended shell linker\n");
+  printf("Include files, resolve commands on build time, process and minify shell code\n");
   printf("\n");
   printf("Options:\n");
   options.print_help(4,25);
@@ -108,10 +136,45 @@ void print_resolve_help()
   printf("Execute shell command and substitute output, from folder of current file\n");
   printf(" - Default behaviour is to parse contents as shell code\n");
   printf(" - Fails if return value is not 0. Can be ignored with -f\n");
-  printf(" - `%%include` inside substitutions replaces the substitution and puts raw response\n");
+  printf(" - `%%resolve` inside substitutions replaces the substitution and puts raw response\n");
   printf("\n");
 
   ztd::option_set opts=create_resolve_opts();
   printf("Options:\n");
   opts.print_help(3,7);
 }
+
+void print_lxsh_extension_help()
+{
+  for(auto it: lxsh_extend_fcts)
+  {
+    printf("%s %s\n%s\n\n", it.first.c_str(), it.second.arguments.c_str(), it.second.description.c_str());
+  }
+}
+
+void oneshot_opt_process(const char* arg0)
+{
+  if(options['h'])
+  {
+    print_help(arg0);
+    exit(ERR_HELP);
+  }
+  else if(options["version"])
+  {
+    printf("%s %s%s\n", arg0, VERSION_STRING, VERSION_SUFFIX);
+    printf("%s\n", VERSION_SHA);
+    exit(0);
+  }
+  else if(options["help-link-commands"])
+  {
+    print_include_help();
+    printf("\n\n");
+    print_resolve_help();
+    exit(ERR_HELP);
+  }
+  else if(options["help-extend-fcts"])
+  {
+    print_lxsh_extension_help();
+    exit(ERR_HELP);
+  }
+}

src/processing.cpp

@@ -5,6 +5,12 @@
 #include "recursive.hpp"
 #include "parse.hpp"
 #include "util.hpp"
+#include "shellcode.hpp"
+#include "struc_helper.hpp"
+#include "options.hpp"
+#include "minify.hpp"
+
+#include "errcodes.h"
 
 // Global regex
 
@@ -56,12 +62,7 @@ void require_rescan_all()
 // type tools
 countmap_t combine_maps(countmap_t const& a, countmap_t const& b)
 {
-  countmap_t ret;
-  for(auto it: a)
-  {
-    if(!ret.insert( it ).second)
-      ret[it.first] += it.second;
-  }
+  countmap_t ret = a;
   for(auto it: b)
   {
     if(!ret.insert( it ).second)
@@ -70,6 +71,19 @@ countmap_t combine_maps(countmap_t const& a, countmap_t const& b)
   return ret;
 }
 
+// add the values of b to a only if they are already present in a
+countmap_t combine_common(countmap_t const& a, countmap_t const& b)
+{
+  countmap_t ret = a;
+  for(auto it: a)
+  {
+    auto t=b.find(it.first);
+    if(t!=b.end())
+      ret[it.first] += t->second;
+  }
+  return ret;
+}
+
 void list_map(countmap_t const& map)
 {
   uint32_t max=0;
@@ -141,7 +155,7 @@ std::string get_varname(std::string const& in)
     return in;
 }
 
-std::string get_varname(arg* in)
+std::string get_varname(arg_t* in)
 {
   if(in->sa.size() < 1 || in->sa[0]->type != _obj::subarg_string)
     return "";
@@ -153,17 +167,17 @@ std::string get_varname(arg* in)
 
 bool cmd_is_argvar(std::string const& in)
 {
-  return in == "export" || in == "unset" || in == "local" || in == "read";
+  return is_in_set(in, posix_cmdvar) || is_in_set(in, bash_cmdvar);
 }
 
-bool cmd::is_argvar()
+bool cmd_t::is_argvar()
 {
-  return cmd_is_argvar(this->firstarg_string());
+  return is_cmdvar;
 }
 
-bool cmd::is(std::string const& in)
+bool cmd_t::is(std::string const& in)
 {
-  return in == this->firstarg_string();
+  return in == this->arg_string(0);
 }
 
 /** GETTERS **/
@@ -199,6 +213,39 @@ void cmdmap_get(_obj* in, std::regex const& exclude)
   }
 }
 
+void fctcmdmap_get(_obj* in, std::regex const& exclude_fct, std::regex const& exclude_cmd)
+{
+  if(!b_gotcmd && !b_gotfct) {
+    b_gotcmd = b_gotfct = true;
+    recurse(r_get_fctcmd, in, &m_cmds, &m_fcts);
+    m_excluded_fct = prune_matching(m_cmds, exclude_cmd);
+    concat_sets(m_excluded_fct, prune_matching(m_fcts, exclude_fct));
+  }
+  else {
+    cmdmap_get(in, exclude_fct);
+    fctmap_get(in, exclude_cmd);
+  }
+}
+
+void allmaps_get(_obj* in, std::regex const& exclude_var, std::regex const& exclude_fct, std::regex const& exclude_cmd)
+{
+  if(!b_gotvar && !b_gotcmd && !b_gotfct)
+  {
+    b_gotvar = b_gotcmd = b_gotfct = true;
+    recurse(r_get_all, in, &m_vardefs, &m_varcalls, &m_cmds, &m_fcts);
+    m_excluded_fct = prune_matching(m_cmds, exclude_cmd);
+    concat_sets(m_excluded_fct, prune_matching(m_fcts, exclude_fct));
+    m_vars = combine_maps(m_vardefs, m_varcalls);
+    m_excluded_var = prune_matching(m_vars, exclude_var);
+  }
+  else
+  {
+    varmap_get(in, exclude_var);
+    cmdmap_get(in, exclude_fct);
+    fctmap_get(in, exclude_fct);
+  }
+}
+
 /** OUTPUT **/
 
 void list_vars(_obj* in, std::regex const& exclude)
@@ -238,51 +285,62 @@ void add_unset_variables(shmain* sh, std::regex const& exclude)
   varmap_get(sh, exclude);
   if(m_vars.size()>0)
   {
-    cmd* unset_cmd = new cmd;
-    unset_cmd->add_arg(new arg("unset"));
+    cmd_t* unset_cmd = new cmd_t;
+    unset_cmd->add(new arg_t("unset"));
+    unset_cmd->is_cmdvar=true;
     for(auto it: m_vars)
-      unset_cmd->add_arg(new arg(it.first));
-    condlist* cl = new condlist(unset_cmd);
+    {
+      unset_cmd->cmd_var_assigns.push_back(std::make_pair(new variable_t(it.first), nullptr));
+    }
+    condlist_t* cl = new condlist_t(unset_cmd);
     sh->lst->cls.insert(sh->lst->cls.begin(), cl);
   }
 }
 
+bool has_env_set(_obj* in) {
+  bool r=false;
+  recurse(r_has_env_set, in, &r);
+  return r;
+}
+
 /** RECURSIVES **/
 
+// CHECK //
+
+bool r_has_env_set(_obj* in, bool* result)
+{
+  switch(in->type)
+  {
+    case _obj::block_subshell: {
+      return false;
+    }; break;
+    case _obj::block_cmd: {
+      cmd_t* t = dynamic_cast<cmd_t*>(in);
+      if(t->has_var_assign() || t->arg_string(0) == "cd")
+        *result = true;
+    }
+    default: break;
+  }
+  return true;
+}
+
 // GET //
 
 bool r_get_var(_obj* in, countmap_t* defmap, countmap_t* callmap)
 {
   switch(in->type)
   {
-    case _obj::subarg_variable: {
-      variable_subarg* t = dynamic_cast<variable_subarg*>(in);
-      if(!callmap->insert( std::make_pair(t->varname, 1) ).second)
-        (*callmap)[t->varname]++;
-    }; break;
-    case _obj::subarg_manipulation: {
-      manipulation_subarg* t = dynamic_cast<manipulation_subarg*>(in);
-      if(!callmap->insert( std::make_pair(t->varname, 1) ).second)
-        (*callmap)[t->varname]++;
-    }; break;
-    case _obj::block_for: {
-      for_block* t = dynamic_cast<for_block*>(in);
-      if(!defmap->insert( std::make_pair(t->varname, 1) ).second)
-        (*defmap)[t->varname]++;
-    }; break;
-    case _obj::block_cmd: {
-      cmd* t = dynamic_cast<cmd*>(in);
-      for(auto it: t->var_assigns)
-        if(!defmap->insert( std::make_pair(it.first, 1) ).second)
-          (*defmap)[it.first]++;
-      if(t->is_argvar())
+    case _obj::variable: {
+      variable_t* t = dynamic_cast<variable_t*>(in);
+      if(t->definition)
       {
-        for(uint32_t i=1; i<t->args->size(); i++)
-        {
-          std::string varname=get_varname(t->args->args[i]);
-          if( varname != "" && !defmap->insert( std::make_pair(varname, 1) ).second )
-            (*defmap)[varname]++;
-        }
+        if(!defmap->insert( std::make_pair(t->varname, 1) ).second)
+          (*defmap)[t->varname]++;
+      }
+      else
+      {
+        if(!callmap->insert( std::make_pair(t->varname, 1) ).second)
+          (*callmap)[t->varname]++;
       }
     }; break;
     default: break;
@@ -295,12 +353,13 @@ bool r_get_unsets(_obj* in, set_t* unsets)
   switch(in->type)
   {
     case _obj::block_cmd: {
-      cmd* t = dynamic_cast<cmd*>(in);
+      cmd_t* t = dynamic_cast<cmd_t*>(in);
       if(t->is("unset"))
       {
-        for(uint32_t i=1; i<t->args->size(); i++)
+        for(auto it: t->cmd_var_assigns)
         {
-          unsets->insert(t->args->args[i]->string());
+          if(it.first != nullptr)
+            unsets->insert(it.first->varname);
         }
       }
     }; break;
@@ -314,8 +373,8 @@ bool r_get_cmd(_obj* in, countmap_t* all_cmds)
   switch(in->type)
   {
     case _obj::block_cmd: {
-      cmd* t = dynamic_cast<cmd*>(in);
-      std::string cmdname = t->firstarg_string();
+      cmd_t* t = dynamic_cast<cmd_t*>(in);
+      std::string cmdname = t->arg_string(0);
       if(cmdname != "" && !all_cmds->insert( std::make_pair(cmdname, 1) ).second)
         (*all_cmds)[cmdname]++;
     }; break;
@@ -329,7 +388,7 @@ bool r_get_fct(_obj* in, countmap_t* fct_map)
   switch(in->type)
   {
     case _obj::block_function: {
-      function* t = dynamic_cast<function*>(in);
+      function_t* t = dynamic_cast<function_t*>(in);
       if(!fct_map->insert( std::make_pair(t->name, 1) ).second)
         (*fct_map)[t->name]++;
     }; break;
@@ -338,20 +397,35 @@ bool r_get_fct(_obj* in, countmap_t* fct_map)
   return true;
 }
 
+bool r_get_fctcmd(_obj* in, countmap_t* all_cmds, countmap_t* fct_map)
+{
+  r_get_cmd(in, all_cmds);
+  r_get_fct(in, fct_map);
+  return true;
+}
+
+bool r_get_all(_obj* in, countmap_t* defmap, countmap_t* callmap, countmap_t* all_cmds, countmap_t* fct_map)
+{
+  r_get_var(in, defmap, callmap);
+  r_get_cmd(in, all_cmds);
+  r_get_fct(in, fct_map);
+  return true;
+}
+
 // DELETE //
 
 bool r_delete_fct(_obj* in, set_t* fcts)
 {
   switch(in->type)
   {
-    case _obj::_list: {
-      list* t = dynamic_cast<list*>(in);
+    case _obj::list: {
+      list_t* t = dynamic_cast<list_t*>(in);
       for(uint32_t i=0; i<t->cls.size(); i++)
       {
-        block* tb = t->cls[i]->first_block();
+        block_t* tb = t->cls[i]->first_block();
         if(tb != nullptr && tb->type == _obj::block_function)
         {
-          function* fc = dynamic_cast<function*>(tb);
+          function_t* fc = dynamic_cast<function_t*>(tb);
           if(fcts->find(fc->name)!=fcts->end())
           {
             delete t->cls[i];
@@ -370,42 +444,47 @@ bool r_delete_var(_obj* in, set_t* vars)
 {
   switch(in->type)
   {
-    case _obj::_list: {
-      list* t = dynamic_cast<list*>(in);
+    case _obj::list: {
+      list_t* t = dynamic_cast<list_t*>(in);
       for(uint32_t i=0; i<t->cls.size(); i++)
       {
-        block* tb = t->cls[i]->first_block();
+        block_t* tb = t->cls[i]->first_block();
         bool to_delete=false;
+        bool has_deleted=false;
         if(tb != nullptr && tb->type == _obj::block_cmd)
         {
-          cmd* c = dynamic_cast<cmd*>(tb);
+          cmd_t* c = dynamic_cast<cmd_t*>(tb);
 
           for(uint32_t j=0; j<c->var_assigns.size(); j++)
           {
-            if( vars->find(c->var_assigns[j].first) != vars->end() )
+            if( c->var_assigns[j].first != nullptr && vars->find(c->var_assigns[j].first->varname) != vars->end() )
             {
-              delete c->var_assigns[j].second;
+              if(c->var_assigns[j].first != nullptr)
+                delete c->var_assigns[j].first;
+              if(c->var_assigns[j].second != nullptr)
+                delete c->var_assigns[j].second;
               c->var_assigns.erase(c->var_assigns.begin()+j);
+              has_deleted=true;
               j--;
             }
           }
+          if(has_deleted && c->var_assigns.size()<=0 && (c->arglist_size()<=0 || c->is_cmdvar) )
+            to_delete=true;
 
-          if(c->is_argvar())
+          for(uint32_t j=0; j<c->cmd_var_assigns.size(); j++)
           {
-            for(uint32_t j=1; j<c->args->size(); j++)
+            if( c->cmd_var_assigns[j].first != nullptr && vars->find(c->cmd_var_assigns[j].first->varname) != vars->end() )
             {
-              std::string varname=get_varname(c->args->args[j]);
-              if( varname != "" && vars->find( varname ) != vars->end() )
-              {
-                delete c->args->args[j];
-                c->args->args.erase(c->args->args.begin()+j);
-                j--;
-              }
+              if(c->cmd_var_assigns[j].first != nullptr)
+                delete c->cmd_var_assigns[j].first;
+              if(c->cmd_var_assigns[j].second != nullptr)
+                delete c->cmd_var_assigns[j].second;
+              c->cmd_var_assigns.erase(c->cmd_var_assigns.begin()+j);
+              has_deleted=true;
+              j--;
             }
-            if(c->args->size()<=1)
-              to_delete=true;
           }
-          if(c->var_assigns.size()<=0 && c->arglist_size()<=0)
+          if(has_deleted && c->cmd_var_assigns.size()<=0 && (c->arglist_size()<=0 || c->is_cmdvar) )
             to_delete=true;
 
         }
@@ -415,9 +494,475 @@ bool r_delete_var(_obj* in, set_t* vars)
           t->cls.erase(t->cls.begin()+i);
           i--;
         }
+        if(t->cls.size()<=0)
+          t->add(make_condlist("true"));
       }
     }
     default: break;
   }
   return true;
 }
+
+bool r_delete_varfct(_obj* in, set_t* vars, set_t* fcts)
+{
+  r_delete_var(in, vars);
+  r_delete_fct(in, fcts);
+  return true;
+}
+
+std::set<std::string> find_lxsh_commands(shmain* sh)
+{
+  std::set<std::string> ret;
+  cmdmap_get(sh, regex_null);
+  for(auto it: lxsh_extend_fcts)
+  {
+    if(m_cmds.find(it.first) != m_cmds.end())
+    {
+      ret.insert(it.first);
+    }
+  }
+  return ret;
+}
+
+std::set<std::string> get_processors(std::string const& in)
+{
+  std::set<std::string> ret;
+  if(in.size()>2 && in[0] == '\'' && in[in.size()-1] == '\'')
+  {
+    uint32_t i=1;
+    while(true)
+    {
+      std::string ln = in.substr(i, in.find('\n', i)-i);
+      if(ln.size()>1 && ln[0] == '#' && is_alphanum(ln[1]))
+      {
+        i+=ln.size();
+        ret.insert(get_word(make_context(ln.substr(1)), SEPARATORS).first);
+      }
+      else
+        break;
+    }
+  }
+  return ret;
+}
+
+bool r_do_string_processor(_obj* in)
+{
+  if(in->type == _obj::subarg_string)
+  {
+    subarg_string_t* t = dynamic_cast<subarg_string_t*>(in);
+    auto v = get_processors(t->val);
+    if(v.find("LXSH_PARSE_MINIFY") != v.end())
+    {
+      try
+      {
+        std::string stringcode = t->val.substr(1, t->val.size()-2);
+        shmain* tsh = parse_text( stringcode ).first;
+        require_rescan_all();
+        if(options["remove-unused"])
+          delete_unused( tsh, re_var_exclude, re_fct_exclude );
+        if(options["minify"])
+          minify_generic(tsh);
+        if(options["minify-var"])
+          minify_var( tsh, re_var_exclude );
+        if(options["minify-fct"])
+          minify_fct( tsh, re_fct_exclude );
+        require_rescan_all();
+        t->val='\'' + tsh->generate(false, 0) + '\'';
+      }
+      catch(format_error& e) // if fail: skip processing
+      {
+        std::cerr << "Exception caused in string processing LXSH_PARSE_MINIFY\n";
+        printFormatError(e);
+        exit(ERR_RUNTIME);
+      }
+    }
+  }
+  return true;
+}
+
+void string_processors(_obj* in)
+{
+  // recurse(r_do_string_processor, in);
+  ;
+}
+
+/** JSON **/
+
+#ifdef DEBUG_MODE
+
+std::string quote_string(std::string const& in)
+{
+  return '"' + stringReplace(stringReplace(stringReplace(in, "\\", "\\\\"), "\"", "\\\""), "\n", "\\n") + '"';
+}
+
+std::string gen_json(std::vector<std::pair<std::string,std::string>> const& vec)
+{
+  std::string ret;
+  for(auto it: vec)
+  {
+    ret += it.first + ":" + it.second + ',';
+  }
+  if(ret != "")
+    ret.pop_back();
+  return "{" + ret + "}";
+}
+
+std::string gen_json(std::vector<std::string> const& vec)
+{
+  std::string ret;
+  for(auto it: vec)
+  {
+    ret += it + ',';
+  }
+  if(ret != "")
+    ret.pop_back();
+  return "[" + ret + "]";
+}
+
+std::string boolstring(bool in)
+{
+  if(in)
+    return "true";
+  else
+    return "false";
+}
+
+std::string gen_json_struc(_obj* o)
+{
+  if(o==nullptr)
+    return "{}";
+  std::vector<std::pair<std::string,std::string>> vec;
+  switch(o->type)
+  {
+    case _obj::variable :
+    {
+      variable_t* t = dynamic_cast<variable_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("variable") ) );
+      vec.push_back(std::make_pair(quote_string("varname"), quote_string(t->varname)));
+      vec.push_back(std::make_pair(quote_string("definition"), boolstring(t->definition)));
+      vec.push_back(std::make_pair(quote_string("index"), gen_json_struc(t->index)));
+      vec.push_back(std::make_pair(quote_string("is_manip"), boolstring(t->is_manip) ) );
+      vec.push_back(std::make_pair(quote_string("precedence"), boolstring(t->precedence) ) );
+      vec.push_back(std::make_pair(quote_string("manip"), gen_json_struc(t->manip) ) );
+      break;
+    }
+    case _obj::redirect :
+    {
+      redirect_t* t = dynamic_cast<redirect_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("redirect") ) );
+      vec.push_back(std::make_pair(quote_string("op"), quote_string(t->op)));
+      vec.push_back(std::make_pair(quote_string("target"), gen_json_struc(t->target)));
+      vec.push_back(std::make_pair(quote_string("here_document"), gen_json_struc(t->here_document)));
+      break;
+    }
+    case _obj::arg :
+    {
+      arg_t* t = dynamic_cast<arg_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("arg") ) );
+      vec.push_back(std::make_pair(quote_string("forcequoted"), boolstring(t->forcequoted)));
+      std::vector<std::string> tvec;
+      for(auto it: t->sa)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("sa"), gen_json(tvec)));
+      break;
+    }
+    case _obj::arglist :
+    {
+      arglist_t* t = dynamic_cast<arglist_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("arglist") ) );
+      std::vector<std::string> tvec;
+      for(auto it: t->args)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("args"), gen_json(tvec)));
+      break;
+    }
+    case _obj::pipeline :
+    {
+      pipeline_t* t = dynamic_cast<pipeline_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("pipeline") ) );
+      vec.push_back(std::make_pair(quote_string("negated"), boolstring(t->negated) ) );
+      std::vector<std::string> tvec;
+      for(auto it: t->cmds)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("cmds"), gen_json(tvec)));
+      break;
+    }
+    case _obj::condlist :
+    {
+      condlist_t* t = dynamic_cast<condlist_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("condlist") ) );
+      vec.push_back(std::make_pair(quote_string("parallel"), boolstring(t->parallel) ) );
+      std::vector<std::string> tvec;
+      for(auto it: t->pls)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("pls"), gen_json(tvec)));
+
+      std::vector<std::string> ttvec;
+      for(auto it: t->or_ops)
+      {
+        ttvec.push_back(boolstring(it));
+      }
+      vec.push_back(std::make_pair(quote_string("or_ops"), gen_json(ttvec)));
+      break;
+    }
+    case _obj::list :
+    {
+      list_t* t = dynamic_cast<list_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("list") ) );
+      std::vector<std::string> tvec;
+      for(auto it: t->cls)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("cls"), gen_json(tvec)));
+      break;
+    }
+    case _obj::block_subshell :
+    {
+      subshell_t* t = dynamic_cast<subshell_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("subshell") ) );
+
+      vec.push_back(std::make_pair(quote_string("lst"), gen_json_struc(t->lst)));
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::block_brace :
+    {
+      brace_t* t = dynamic_cast<brace_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("brace") ) );
+
+      vec.push_back(std::make_pair(quote_string("lst"), gen_json_struc(t->lst)));
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::block_main :
+    {
+      shmain* t = dynamic_cast<shmain*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("main") ) );
+      vec.push_back(std::make_pair(quote_string("shebang"), quote_string(t->shebang) ) );
+
+      vec.push_back(std::make_pair(quote_string("lst"), gen_json_struc(t->lst)));
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::block_function :
+    {
+      function_t* t = dynamic_cast<function_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("function") ) );
+      vec.push_back(std::make_pair(quote_string("name"), quote_string(t->name) ) );
+
+      vec.push_back(std::make_pair(quote_string("lst"), gen_json_struc(t->lst)));
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::block_cmd :
+    {
+      cmd_t* t = dynamic_cast<cmd_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("cmd") ) );
+
+      vec.push_back(std::make_pair(quote_string("args"), gen_json_struc(t->args)));
+      vec.push_back(std::make_pair(quote_string("is_cmdvar"), boolstring(t->is_cmdvar)));
+
+      std::vector<std::string> aa;
+      for(auto it: t->var_assigns)
+      {
+        std::vector<std::pair<std::string,std::string>> ttvec;
+        ttvec.push_back( std::make_pair(quote_string("var"), gen_json_struc(it.first)) );
+        ttvec.push_back( std::make_pair(quote_string("value"), gen_json_struc(it.second)) );
+        aa.push_back(gen_json(ttvec));
+      }
+      vec.push_back(std::make_pair( quote_string("var_assigns"), gen_json(aa)));
+      std::vector<std::string> bb;
+      for(auto it: t->cmd_var_assigns)
+      {
+        std::vector<std::pair<std::string,std::string>> ttvec;
+        ttvec.push_back( std::make_pair(quote_string("var"), gen_json_struc(it.first)) );
+        ttvec.push_back( std::make_pair(quote_string("value"), gen_json_struc(it.second)) );
+        bb.push_back(gen_json(ttvec));
+      }
+      vec.push_back(std::make_pair( quote_string("cmd_var_assigns"), gen_json(bb)));
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::block_case :
+    {
+      case_t* t = dynamic_cast<case_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("case") ) );
+
+      vec.push_back(std::make_pair(quote_string("carg"), gen_json_struc(t->carg)));
+
+      // [ {matchers:[], exec:{}} ]
+      // cases
+      std::vector<std::string> tt;
+      for(auto sc: t->cases)
+      {
+        std::vector<std::pair<std::string,std::string>> onecase;
+        std::vector<std::string> matchers;
+        for(auto it: sc.first)
+          matchers.push_back(gen_json_struc(it));
+        onecase.push_back( std::make_pair(quote_string("matcher"), gen_json(matchers)) );
+        onecase.push_back( std::make_pair(quote_string("execution"), gen_json_struc(sc.second)) );
+        tt.push_back(gen_json(onecase));
+      }
+      vec.push_back( std::make_pair(quote_string("cases"),gen_json(tt)) );
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::block_if :
+    {
+      if_t* t = dynamic_cast<if_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("if") ) );
+
+      std::vector<std::string> condblocks;
+      // ifs
+      for(auto sc: t->blocks)
+      {
+        std::vector<std::pair<std::string, std::string> > one_cond;
+        one_cond.push_back(std::make_pair(quote_string("condition"), gen_json_struc(sc.first) ) );
+        one_cond.push_back(std::make_pair(quote_string("execution"), gen_json_struc(sc.second) ) );
+        condblocks.push_back(gen_json(one_cond));
+      }
+      vec.push_back( std::make_pair(quote_string("blocks"), gen_json(condblocks)) );
+      // else
+      vec.push_back(std::make_pair(quote_string("else_lst"), gen_json_struc(t->else_lst)));
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::block_for :
+    {
+      for_t* t = dynamic_cast<for_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("for") ) );
+      vec.push_back(std::make_pair(quote_string("var"), gen_json_struc(t->var)));
+      vec.push_back(std::make_pair(quote_string("iter"), gen_json_struc(t->iter)));
+      vec.push_back(std::make_pair(quote_string("ops"), gen_json_struc(t->ops)));
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::block_while :
+    {
+      while_t* t = dynamic_cast<while_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("while") ) );
+      vec.push_back(std::make_pair(quote_string("cond"), gen_json_struc(t->cond) ) );
+      vec.push_back(std::make_pair(quote_string("ops"), gen_json_struc(t->ops) ) );
+
+      std::vector<std::string> tvec;
+      for(auto it: t->redirs)
+        tvec.push_back(gen_json_struc(it));
+      vec.push_back(std::make_pair(quote_string("redirs"), gen_json(tvec)));
+
+      break;
+    }
+    case _obj::subarg_variable :
+    {
+      subarg_variable_t* t = dynamic_cast<subarg_variable_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("subarg_variable") ) );
+      vec.push_back(std::make_pair(quote_string("var"), gen_json_struc(t->var) ) );
+      break;
+    }
+    case _obj::subarg_subshell :
+    {
+      subarg_subshell_t* t = dynamic_cast<subarg_subshell_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("subarg_subshell") ) );
+      vec.push_back(std::make_pair(quote_string("sbsh"), gen_json_struc(t->sbsh) ) );
+      break;
+    }
+    case _obj::subarg_procsub :
+    {
+      subarg_procsub_t* t = dynamic_cast<subarg_procsub_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("subarg_procsub") ) );
+      vec.push_back(std::make_pair(quote_string("is_output"), boolstring(t->is_output) ) );
+      vec.push_back(std::make_pair(quote_string("sbsh"), gen_json_struc(t->sbsh) ) );
+      break;
+    }
+    case _obj::subarg_arithmetic :
+    {
+      subarg_arithmetic_t* t = dynamic_cast<subarg_arithmetic_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("subarg_arithmetic") ) );
+      vec.push_back(std::make_pair(quote_string("arith"), gen_json_struc(t->arith) ) );
+      break;
+    }
+    case _obj::subarg_string :
+    {
+      subarg_string_t* t = dynamic_cast<subarg_string_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("subarg_string") ) );
+      vec.push_back(std::make_pair(quote_string("val"), quote_string(t->val) ) );
+      break;
+    }
+    case _obj::arithmetic_variable :
+    {
+      arithmetic_variable_t* t = dynamic_cast<arithmetic_variable_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("arithmetic_variable") ) );
+      vec.push_back(std::make_pair(quote_string("var"), gen_json_struc(t->var) ) );
+      break;
+    }
+    case _obj::arithmetic_subshell :
+    {
+      arithmetic_subshell_t* t = dynamic_cast<arithmetic_subshell_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("arithmetic_subshell") ) );
+      vec.push_back(std::make_pair(quote_string("sbsh"), gen_json_struc(t->sbsh) ) );
+      break;
+    }
+    case _obj::arithmetic_operation :
+    {
+      arithmetic_operation_t* t = dynamic_cast<arithmetic_operation_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("arithmetic_operation") ) );
+      vec.push_back(std::make_pair(quote_string("val1"), gen_json_struc(t->val1) ) );
+      vec.push_back(std::make_pair(quote_string("val2"), gen_json_struc(t->val2) ) );
+      break;
+    }
+    case _obj::arithmetic_parenthesis :
+    {
+      arithmetic_parenthesis_t* t = dynamic_cast<arithmetic_parenthesis_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("arithmetic_parenthesis") ) );
+      vec.push_back(std::make_pair(quote_string("val"), gen_json_struc(t->val) ) );
+      break;
+    }
+    case _obj::arithmetic_number :
+    {
+      arithmetic_number_t* t = dynamic_cast<arithmetic_number_t*>(o);
+      vec.push_back(std::make_pair(quote_string("type"), quote_string("arithmetic_number") ) );
+      vec.push_back(std::make_pair(quote_string("val"), quote_string(t->val) ) );
+      break;
+    }
+  }
+  return gen_json(vec);
+}
+#endif

src/resolve.cpp

@@ -39,12 +39,12 @@ bool add_include(std::string const& file)
 }
 
 // returns path to old dir
-std::string _pre_cd(shmain* parent)
+std::string _pre_cd(std::string const& filename)
 {
-  if(parent->is_dev_file() || parent->filename == "")
+  if(filename == "" || is_dev_file(filename))
     return "";
   std::string dir=pwd();
-  std::string cddir=dirname(parent->filename);
+  std::string cddir=dirname(filename);
   if(chdir(cddir.c_str()) != 0)
     throw std::runtime_error("Cannot cd to '"+cddir+"'");
   return dir;
@@ -59,7 +59,7 @@ void _cd(std::string const& dir)
 // -- COMMANDS --
 
 // return <name, contents>[]
-std::vector<std::pair<std::string, std::string>> do_include_raw(condlist* cmd, shmain* parent, std::string* ex_dir=nullptr)
+std::vector<std::pair<std::string, std::string>> do_include_raw(condlist_t* cmd, parse_context ctx, std::string* ex_dir)
 {
   std::vector<std::pair<std::string, std::string>> ret;
 
@@ -67,7 +67,7 @@ std::vector<std::pair<std::string, std::string>> do_include_raw(condlist* cmd, s
   std::vector<std::string> rargs;
   try
   {
-    rargs = opts.process(cmd->first_cmd()->args->strargs(1), false, true, false);
+    rargs = opts.process(cmd->first_cmd()->args->strargs(1), {.stop_on_argument=true});
   }
   catch(ztd::option_error& e)
   {
@@ -77,7 +77,7 @@ std::vector<std::pair<std::string, std::string>> do_include_raw(condlist* cmd, s
   std::string dir;
   if(g_cd && !opts['C'])
   {
-    dir=_pre_cd(parent);
+    dir=_pre_cd(ctx.filename);
     if(ex_dir!=nullptr)
       *ex_dir=dir;
   }
@@ -104,31 +104,8 @@ std::vector<std::pair<std::string, std::string>> do_include_raw(condlist* cmd, s
   return ret;
 }
 
-std::vector<condlist*> do_include_parse(condlist* cmd, shmain* parent)
-{
-  std::vector<condlist*> ret;
-
-  std::string dir;
-  auto incs=do_include_raw(cmd, parent, &dir);
-
-  for(auto it: incs)
-  {
-    shmain* sh=parse_text(it.second, it.first);
-    resolve(sh);
-    // get the cls
-    ret.insert(ret.end(), sh->lst->cls.begin(), sh->lst->cls.end());
-    // safety and cleanup
-    sh->lst->cls.resize(0);
-    delete sh;
-  }
-  // cd back
-  _cd(dir);
-
-  return ret;
-}
-
 //
-std::pair<std::string, std::string> do_resolve_raw(condlist* cmd, shmain* parent, std::string* ex_dir=nullptr)
+std::pair<std::string, std::string> do_resolve_raw(condlist_t* cmd, parse_context ctx, std::string* ex_dir)
 {
   std::pair<std::string, std::string> ret;
 
@@ -136,7 +113,7 @@ std::pair<std::string, std::string> do_resolve_raw(condlist* cmd, shmain* parent
   std::vector<std::string> rargs;
   try
   {
-    rargs = opts.process(cmd->first_cmd()->args->strargs(1), false, true, false);
+    rargs = opts.process(cmd->first_cmd()->args->strargs(1), {.stop_on_argument=true} );
   }
   catch(ztd::option_error& e)
   {
@@ -146,7 +123,7 @@ std::pair<std::string, std::string> do_resolve_raw(condlist* cmd, shmain* parent
   std::string dir;
   if(g_cd && !opts['C'])
   {
-    dir=_pre_cd(parent);
+    dir=_pre_cd(ctx.filename);
     if(ex_dir!=nullptr)
       *ex_dir=dir;
   }
@@ -175,20 +152,56 @@ std::pair<std::string, std::string> do_resolve_raw(condlist* cmd, shmain* parent
   return ret;
 }
 
-// if first is nullptr: is a string
-std::vector<condlist*> do_resolve_parse(condlist* cmd, shmain* parent)
+std::vector<condlist_t*> do_include_parse(condlist_t* cmd, parse_context ctx)
 {
-  std::vector<condlist*> ret;
+  std::vector<condlist_t*> ret;
+
+  std::string dir;
+  auto incs=do_include_raw(cmd, ctx, &dir);
+
+  std::vector<shmain*> shs;
+  shs.resize(incs.size());
+
+  for(uint32_t i=0; i<incs.size(); i++)
+  {
+    parse_context newctx = make_context(ctx, incs[i].second, incs[i].first);
+    auto pp = parse_text(newctx);
+    shmain* sh = pp.first;
+    resolve(sh, pp.second);
+    shs[i] = sh;
+  }
+  for(auto sh: shs)
+  {
+    // get the cls
+    ret.insert(ret.end(), sh->lst->cls.begin(), sh->lst->cls.end());
+    // safety and cleanup
+    sh->lst->cls.resize(0);
+    delete sh;
+  }
+  shs.resize(0);
+  // cd back
+  _cd(dir);
+
+  return ret;
+}
+
+// if first is nullptr: is a string
+std::vector<condlist_t*> do_resolve_parse(condlist_t* cmd, parse_context ctx)
+{
+  std::vector<condlist_t*> ret;
 
   std::pair<std::string,std::string> p;
   try
   {
     // get
     std::string dir;
-    p=do_resolve_raw(cmd, parent, &dir);
+    p=do_resolve_raw(cmd, ctx, &dir);
+
     // do parse
-    shmain* sh = parse_text(p.second);
-    resolve(sh);
+    parse_context newctx = make_context(ctx, p.second, '`'+p.first+'`');
+    auto pp = parse_text(newctx);
+    shmain* sh = pp.first;
+    resolve(sh, pp.second);
     // get the cls
     ret = sh->lst->cls;
     // safety and cleanup
@@ -197,9 +210,9 @@ std::vector<condlist*> do_resolve_parse(condlist* cmd, shmain* parent)
     // cd back
     _cd(dir);
   }
-  catch(ztd::format_error& e)
+  catch(format_error& e)
   {
-    throw ztd::format_error(e.what(), '`'+p.first+'`', e.data(), e.where());
+    throw format_error(e.what(), '`'+p.first+'`', e.data(), e.where());
   }
 
   return ret;
@@ -207,50 +220,54 @@ std::vector<condlist*> do_resolve_parse(condlist* cmd, shmain* parent)
 
 // -- OBJECT CALLS --
 
-std::pair< std::vector<condlist*> , bool > resolve_condlist(condlist* in, shmain* parent)
+std::pair< std::vector<condlist_t*> , bool > resolve_condlist(condlist_t* in, parse_context ctx)
 {
-  cmd* tc = in->first_cmd();
+  cmd_t* tc = in->first_cmd();
   if(tc == nullptr)
-    return std::make_pair(std::vector<condlist*>(), false);
+    return std::make_pair(std::vector<condlist_t*>(), false);
 
-  std::string const& strcmd=tc->firstarg_string();
+  std::string const& strcmd=tc->arg_string(0);
 
   if(g_include && strcmd == "%include")
-    return std::make_pair(do_include_parse(in, parent), true);
+    return std::make_pair(do_include_parse(in, ctx), true);
   else if(g_resolve && strcmd == "%resolve")
-    return std::make_pair(do_resolve_parse(in, parent), true);
+    return std::make_pair(do_resolve_parse(in, ctx), true);
   else
-    return std::make_pair(std::vector<condlist*>(), false);
+    return std::make_pair(std::vector<condlist_t*>(), false);
 }
 
-std::pair< std::vector<arg*> , bool > resolve_arg(arg* in, shmain* parent, bool forcequote=false)
+std::pair< std::vector<arg_t*> , bool > resolve_arg(arg_t* in, parse_context ctx, bool forcequote=false)
 {
-  std::vector<arg*> ret;
-  arg* ta=nullptr;
+  std::vector<arg_t*> ret;
+  if(in == nullptr)
+  {
+    return std::make_pair(ret, false);
+  }
+  arg_t* ta=nullptr;
   bool has_resolved=false;
   uint32_t j=0;
-  for(uint32_t i=0 ; i<in->sa.size() ; i++)
+  for(uint32_t i=0 ; i<in->size() ; i++)
   {
     if(in->sa[i]->type != _obj::subarg_subshell) // skip if not subshell
       continue;
 
-    subshell_subarg* tsh = dynamic_cast<subshell_subarg*>(in->sa[i]);
-    if(tsh->sbsh->lst->cls.size() > 1) // skip if more than one cl
+    subarg_subshell_t* tsh = dynamic_cast<subarg_subshell_t*>(in->sa[i]);
+    if(tsh->sbsh->lst->cls.size() != 1) // skip if not one cl
       continue;
-    condlist* tc = tsh->sbsh->lst->cls[0];
-    cmd* c = tc->first_cmd();
+    condlist_t* tc = tsh->sbsh->lst->cls[0];
+    cmd_t* c = tc->first_cmd();
     if(c == nullptr) // skip if not cmd
       continue;
-    std::string strcmd=c->firstarg_string();
+    std::string strcmd=c->arg_string(0);
     std::string fulltext;
     if(g_include && strcmd == "%include")
     {
-      for(auto it: do_include_raw(tc, parent) )
+      for(auto it: do_include_raw(tc, ctx) )
         fulltext += it.second;
     }
     else if(g_resolve && strcmd == "%resolve")
     {
-      fulltext = do_resolve_raw(tc, parent).second;
+      fulltext = do_resolve_raw(tc, ctx).second;
     }
     else // skip
       continue;
@@ -260,18 +277,24 @@ std::pair< std::vector<arg*> , bool > resolve_arg(arg* in, shmain* parent, bool
 
     if(tsh->quoted || forcequote)
     {
-      stringReplace(fulltext, "\"", "\\\"");
-      stringReplace(fulltext, "!", "\\!");
+      fulltext = stringReplace(fulltext, "\\\"", "\\\\\"");
+      fulltext = escape_chars(fulltext, "\"`$");
+      fulltext = stringReplace(fulltext, "!", "\"\\!\"");
     }
+    else
+    {
+      fulltext = escape_chars(fulltext, "\\\"!$`#&|()';<>");
+    }
+
     if(!tsh->quoted && forcequote)
-      fulltext = '"' + fulltext + '"';
+     fulltext = '"' + fulltext + '"';
 
 
     if(tsh->quoted || forcequote)
     {
       // replace with new subarg
       delete in->sa[i];
-      in->sa[i] = new string_subarg(fulltext);
+      in->sa[i] = new subarg_string_t(fulltext);
     }
     else
     {
@@ -282,21 +305,21 @@ std::pair< std::vector<arg*> , bool > resolve_arg(arg* in, shmain* parent, bool
         if(strargs.size() == 1)
           val = strargs[0];
         delete in->sa[i];
-        in->sa[i] = new string_subarg(val);
+        in->sa[i] = new subarg_string_t(val);
       }
       else // pack
       {
         if(ta == nullptr)
-          ta = new arg;
+          ta = new arg_t;
         ta->sa.insert(ta->sa.end(), in->sa.begin()+j, in->sa.begin()+i);
-        ta->sa.push_back(new string_subarg(strargs[i]));
+        ta->add(new subarg_string_t(strargs[i]));
         j=i+1;
         delete in->sa[i];
         for(uint32_t li=1 ; li<strargs.size() ; li++)
         {
           ret.push_back(ta);
-          ta = new arg;
-          ta->sa.push_back(new string_subarg(strargs[li]));
+          ta = new arg_t;
+          ta->add(new subarg_string_t(strargs[li]));
         }
 
       } // end pack
@@ -316,10 +339,10 @@ std::pair< std::vector<arg*> , bool > resolve_arg(arg* in, shmain* parent, bool
   return std::make_pair(ret, has_resolved);
 }
 
-
+void resolve(_obj* in, parse_context* ctx);
 // -- RECURSIVE CALL --
 
-bool r_resolve(_obj* o, shmain* parent)
+bool r_resolve(_obj* o, parse_context* ct)
 {
   switch(o->type)
   {
@@ -327,12 +350,12 @@ bool r_resolve(_obj* o, shmain* parent)
     // check every sub-object
     // execute resolve manually
     // instruct parent resolve to not resolve
-    case _obj::_list :
+    case _obj::list :
     {
-      auto t = dynamic_cast<list*>(o);
+      auto t = dynamic_cast<list_t*>(o);
       for(uint32_t i=0 ; i<t->cls.size() ; i++)
       {
-        auto r=resolve_condlist(t->cls[i], parent);
+        auto r=resolve_condlist(t->cls[i], *ct);
         if(r.second)
         {
           // add new cls after current
@@ -345,17 +368,17 @@ bool r_resolve(_obj* o, shmain* parent)
         }
         else
         {
-          resolve(t->cls[i], parent);
+          resolve(t->cls[i], ct);
         }
       }
       return false;
     } break;
-    case _obj::_arglist :
+    case _obj::arglist :
     {
-      auto t = dynamic_cast<arglist*>(o);
-      for(uint32_t i=0 ; i<t->args.size() ; i++)
+      auto t = dynamic_cast<arglist_t*>(o);
+      for(uint32_t i=0 ; i<t->size() ; i++)
       {
-        auto r=resolve_arg(t->args[i], parent);
+        auto r=resolve_arg(t->args[i], *ct);
         if(r.first.size()>0)
         {
           // add new args
@@ -367,38 +390,43 @@ bool r_resolve(_obj* o, shmain* parent)
         }
         else
         {
-          resolve(t->args[i], parent);
+          resolve(t->args[i], ct);
         }
       }
       return false;
     } break;
     case _obj::block_cmd :
     {
-      auto t = dynamic_cast<cmd*>(o);
+      auto t = dynamic_cast<cmd_t*>(o);
       for(auto it: t->var_assigns) // var assigns
       {
-        resolve_arg(it.second, parent, true); // force quoted
-        resolve(it.second, parent);
+        resolve_arg(it.second, *ct, true); // force quoted
+        resolve(it.second, ct);
+      }
+      for(auto it: t->cmd_var_assigns) // var assigns
+      {
+        resolve_arg(it.second, *ct, true); // force quoted
+        resolve(it.second, ct);
       }
       for(auto it: t->redirs)
-        resolve(it, parent);
-      resolve(t->args, parent);
+        resolve(it, ct);
+      resolve(t->args, ct);
       return false;
     }; break;
     case _obj::block_case :
     {
-      auto t = dynamic_cast<case_block*>(o);
+      auto t = dynamic_cast<case_t*>(o);
       for(auto sc: t->cases)
       {
-        resolve_arg(t->carg, parent, true); // force quoted
-        resolve(t->carg, parent);
+        resolve_arg(t->carg, *ct, true); // force quoted
+        resolve(t->carg, ct);
 
         for(auto it: sc.first)
         {
-          resolve_arg(it, parent, true); // force quoted
-          resolve(it, parent);
+          resolve_arg(it, *ct, true); // force quoted
+          resolve(it, ct);
         }
-        resolve(sc.second, parent);
+        resolve(sc.second, ct);
       }
     }; break;
     default: break;
@@ -407,12 +435,13 @@ bool r_resolve(_obj* o, shmain* parent)
 }
 
 // recursive call of resolve
-void resolve(_obj* in, shmain* parent)
+void resolve(_obj* in, parse_context* ctx)
 {
-  recurse(r_resolve, in, parent);
+  recurse(r_resolve, in, ctx);
 }
 
-void resolve(shmain* sh)
+// recursive call of resolve
+void resolve(_obj* in, parse_context ctx)
 {
-  recurse(r_resolve, sh, sh);
+  recurse(r_resolve, in, &ctx);
 }

src/shellcode.cpp

@@ -0,0 +1,49 @@
+#include "shellcode.hpp"
+
+#include "g_shellcode.h"
+#include "processing.hpp"
+#include "struc_helper.hpp"
+
+const std::map<const std::string, const lxsh_fct> lxsh_extend_fcts = {
+    { "_lxsh_random",         { "[K]", "Generate a random number between 0 and 2^(K*8). Default 2", RANDOM_SH} },
+    { "_lxsh_random_string",  { "[N]", "Generate a random alphanumeric string of length N. Default 20", RANDOM_STRING_SH} },
+    { "_lxsh_random_tmpfile", { "[PREFIX] [N]", "Get a random TMP filepath, with N random chars. Default 20", RANDOM_TMPFILE_SH, {"_lxsh_random_string"} }
+  }
+};
+
+const std::map<const std::string, const lxsh_fct> lxsh_array_fcts = {
+  { "_lxsh_array_create",   { "<VAL...>", "Create an array out of input arguments", ARRAY_CREATE_SH} },
+  { "_lxsh_array_get",      { "<ARRAY> <I>",    "Get value from array", ARRAY_GET_SH} },
+  { "_lxsh_array_set",      { "<ARRAY> <I> <VAL>", "Set value of array", ARRAY_SET_SH} },
+  { "_lxsh_map_create",     { "<VAL...>", "Create a map (associative array) out of input arguments", MAP_CREATE_SH} },
+  { "_lxsh_map_get",        { "<MAP> <KEY>",    "Get value from map", MAP_GET_SH} },
+  { "_lxsh_map_set",        { "<MAP> <KEY> <VAL>", "Set value of map", MAP_SET_SH} }
+};
+
+std::map<const std::string, const lxsh_fct> create_allfcts()
+{
+  auto r = lxsh_array_fcts;
+  for(auto it: lxsh_extend_fcts)
+    r.insert(it);
+  return r;
+}
+
+const std::map<const std::string, const lxsh_fct> lxsh_allfcts = create_allfcts();
+
+void add_lxsh_fcts(shmain* sh, std::set<std::string> fcts)
+{
+  // resolve dependencies
+  for(auto fctname: fcts)
+  {
+    auto ti=lxsh_allfcts.find(fctname);
+    if(ti != lxsh_allfcts.end())
+      for(auto dep: ti->second.depends_on)
+        fcts.insert(dep);
+  }
+  // insert functions
+  for(auto it: fcts)
+  {
+    sh->lst->insert(0, make_condlist(lxsh_allfcts.find(it)->second.code) );
+  }
+  require_rescan_all();
+}

src/struc.cpp

@@ -5,148 +5,11 @@
 
 #include <unistd.h>
 
-std::string g_origin="";
+const std::string cmd_t::empty_string="";
 
-const std::string cmd::empty_string="";
-
-std::vector<std::string> arglist::strargs(uint32_t start)
+condlist_t::condlist_t(block_t* bl)
 {
-  std::vector<std::string> ret;
-  bool t=opt_minimize;
-  opt_minimize=true;
-  for(uint32_t i=start; i<args.size(); i++)
-  {
-    ret.push_back(args[i]->generate(0));
-  }
-  opt_minimize=t;
-  return ret;
-}
-
-void arg::setstring(std::string const& str)
-{
-  for(auto it: sa)
-    delete it;
-  sa.resize(0);
-  sa.push_back(new string_subarg(str));
-}
-
-std::string arg::string()
-{
-  if(sa.size() != 1 || sa[0]->type != subarg::subarg_string)
-    return "";
-  return dynamic_cast<string_subarg*>(sa[0])->val;
-}
-
-void condlist::prune_first_cmd()
-{
-  if(pls.size()>0 && pls[0]->cmds.size()>0)
-  {
-    delete pls[0]->cmds[0];
-    pls[0]->cmds.erase(pls[0]->cmds.begin());
-  }
-}
-
-void condlist::add(pipeline* pl, bool or_op)
-{
-  if(this->pls.size() > 0)
-    this->or_ops.push_back(or_op);
-  this->pls.push_back(pl);
-}
-
-block* condlist::first_block()
-{
-  if(pls.size() > 0 && pls[0]->cmds.size() > 0)
-    return (pls[0]->cmds[0]);
-  else
-    return nullptr;
-}
-
-cmd* condlist::first_cmd()
-{
-  if(pls.size() > 0 && pls[0]->cmds.size() > 0 && pls[0]->cmds[0]->type == _obj::block_cmd)
-    return dynamic_cast<cmd*>(pls[0]->cmds[0]);
-  else
-    return nullptr;
-}
-
-cmd* block::single_cmd()
-{
-  if(this->type == _obj::block_subshell)
-  {
-    return dynamic_cast<subshell*>(this)->single_cmd();
-  }
-  if(this->type == _obj::block_brace)
-  {
-    return dynamic_cast<brace*>(this)->single_cmd();
-  }
-  return nullptr;
-}
-
-cmd* subshell::single_cmd()
-{
-  if( lst->size() == 1 && // only one condlist
-      (*lst)[0]->pls.size() == 1 && // only one pipeline
-      (*lst)[0]->pls[0]->cmds.size() == 1 && // only one block
-      (*lst)[0]->pls[0]->cmds[0]->type == _obj::block_cmd) // block is a command
-    return dynamic_cast<cmd*>((*lst)[0]->pls[0]->cmds[0]); // return command
-  return nullptr;
-}
-
-size_t cmd::arglist_size()
-{
-  if(args==nullptr)
-    return 0;
-  else
-    return args->size();
-}
-
-cmd* brace::single_cmd()
-{
-  if( lst->size() == 1 && // only one condlist
-      (*lst)[0]->pls.size() == 1 && // only one pipeline
-      (*lst)[0]->pls[0]->cmds.size() == 1 && // only one block
-      (*lst)[0]->pls[0]->cmds[0]->type == _obj::block_cmd) // block is a command
-    return dynamic_cast<cmd*>((*lst)[0]->pls[0]->cmds[0]); // return command
-  return nullptr;
-}
-cmd* condlist::get_cmd(std::string const& cmdname)
-{
-  for(auto pl: pls)
-  {
-    for(auto bl: pl->cmds)
-    {
-      if(bl->type == _obj::block_cmd)
-      {
-        cmd* c=dynamic_cast<cmd*>(bl);
-        if(c->args->size()>0 && (*c->args)[0]->equals(cmdname) )
-          return c;
-      }
-    }
-  }
-  return nullptr;
-}
-
-void shmain::concat(shmain* in)
-{
-  this->lst->cls.insert(this->lst->cls.end(), in->lst->cls.begin(), in->lst->cls.end());
-  in->lst->cls.resize(0);
-  if(this->shebang == "")
-    this->shebang = in->shebang;
-}
-
-void condlist::negate()
-{
-  // invert commands
-  for(uint32_t i=0; i<pls.size(); i++)
-    pls[i]->negated = !pls[i]->negated;
-  // invert bool operators
-  for(uint32_t i=0; i<or_ops.size(); i++)
-    or_ops[i] = !or_ops[i];
-}
-
-std::string const& cmd::firstarg_string()
-{
-  if(args!=nullptr && args->args.size()>0 && args->args[0]->sa.size() == 1 && args->args[0]->sa[0]->type == _obj::subarg_string)
-    return dynamic_cast<string_subarg*>(args->args[0]->sa[0])->val;
-  return cmd::empty_string;
+  type=_obj::condlist;
+  parallel=false;
+  this->add(new pipeline_t(bl));
 }

src/struc_helper.cpp

@@ -1,34 +1,458 @@
-#include "struc.hpp"
+#include "struc_helper.hpp"
 
-cmd* make_cmd(std::vector<std::string> args)
+#include "parse.hpp"
+#include "options.hpp"
+
+// ** FUNCTIONS ** //
+
+// makers
+
+arg_t* make_arg(std::string const& in)
 {
-  cmd* ret = new cmd();
-  ret->args = new arglist();
+  return parse_arg(make_context(in)).first;
+}
+
+cmd_t* make_cmd(std::vector<const char*> const& args)
+{
+  cmd_t* ret = new cmd_t;
+  ret->args = new arglist_t;
   for(auto it: args)
-  {
-    ret->args->add(new arg(it));
-  }
+    ret->args->add(new arg_t(it));
   return ret;
 }
 
-condlist::condlist(pipeline* pl)
+cmd_t* make_cmd(std::vector<std::string> const& args)
 {
-  type=_obj::_condlist;
-  parallel=false;
-  if(pl!=nullptr) this->add(pl);
+  cmd_t* ret = new cmd_t;
+  ret->args = new arglist_t;
+  for(auto it: args)
+    ret->args->add(new arg_t(it));
+  return ret;
 }
 
-condlist::condlist(block* bl)
+cmd_t* make_cmd(std::vector<arg_t*> const& args)
 {
-  type=_obj::_condlist;
-  parallel=false;
-  this->add(new pipeline(bl));
+  cmd_t* ret = new cmd_t;
+  ret->args = new arglist_t;
+  for(auto it: args)
+    ret->args->add(it);
+
+  return ret;
 }
 
-void cmd::add_arg(arg* in)
+cmd_t* make_cmd(std::string const& in)
+{
+  return parse_cmd(make_context(in)).first;
+}
+
+pipeline_t* make_pipeline(std::vector<block_t*> const& bls)
+{
+  pipeline_t* ret = new pipeline_t;
+  for(auto it: bls)
+    ret->add(it);
+
+  return ret;
+}
+
+pipeline_t* make_pipeline(std::string const& in)
+{
+  return parse_pipeline(make_context(in)).first;
+}
+
+condlist_t* make_condlist(std::string const& in)
+{
+  return parse_condlist(make_context(in)).first;
+}
+
+list_t* make_list(std::string const& in)
+{
+  auto t = parse_list_until(make_context(in));
+  return std::get<0>(t);
+}
+
+block_t* make_block(std::string const& in)
+{
+  return parse_block(make_context(in)).first;
+}
+
+cmd_t* make_printf(arg_t* in)
+{
+  cmd_t* prnt = make_cmd(std::vector<const char*>({"printf", "%s\\\\n"}));
+  force_quotes(in);
+  prnt->add(in);
+  return prnt;
+}
+
+arithmetic_t* make_arithmetic(arg_t* a)
+{
+  if(a->sa.size() != 1)
+  {
+    cmd_t* prnt = make_printf(a);
+    return new arithmetic_subshell_t(new subshell_t(prnt));
+  }
+  arithmetic_t* ret=nullptr;
+  switch(a->sa[0]->type) {
+    case _obj::subarg_string : {
+      subarg_string_t* t = dynamic_cast<subarg_string_t*>(a->sa[0]);
+      ret = new arithmetic_number_t(t->val);
+    }; break;
+    case _obj::subarg_variable : {
+      subarg_variable_t* t = dynamic_cast<subarg_variable_t*>(a->sa[0]);
+      ret = new arithmetic_variable_t(t->var);
+      t->var = nullptr;
+    }; break;
+    case _obj::subarg_subshell : {
+      subarg_subshell_t* t = dynamic_cast<subarg_subshell_t*>(a->sa[0]);
+      ret = new arithmetic_subshell_t(t->sbsh);
+      t->sbsh = nullptr;
+    }; break;
+    case _obj::subarg_arithmetic : {
+      subarg_arithmetic_t* t = dynamic_cast<subarg_arithmetic_t*>(a->sa[0]);
+      ret = t->arith;
+      t->arith = nullptr;
+    }; break;
+    default: break;
+  }
+  delete a;
+  return ret;
+}
+
+arithmetic_t* make_arithmetic(arg_t* arg1, std::string op, arg_t* arg2)
+{
+  return new arithmetic_operation_t(op, make_arithmetic(arg1), make_arithmetic(arg2));
+}
+
+
+// copy
+
+arg_t* copy(arg_t* in) {
+  std::string str = in->generate(0);
+  return parse_arg(make_context(str)).first;
+}
+
+// modifiers
+
+void force_quotes(arg_t* in)
+{
+  for(uint32_t i=0; i < in->sa.size() ; i++)
+  {
+    if(!in->sa[i]->quoted && (in->sa[i]->type == _obj::subarg_variable || in->sa[i]->type == _obj::subarg_subshell) )
+    {
+      in->sa[i]->quoted=true;
+      in->insert(i+1, "\"");
+      in->insert(i, "\"");
+      i+=2;
+    }
+  }
+}
+
+void add_quotes(arg_t* in)
+{
+  for(uint32_t i=0; i < in->sa.size() ; i++)
+    in->sa[i]->quoted=true;
+
+  in->insert(0, new subarg_string_t("\""));
+  in->add("\"");
+}
+
+// ** TESTERS ** //
+
+bool arg_has_char(char c, arg_t* in)
+{
+  for(auto it: in->sa)
+  {
+    if(it->type == _obj::subarg_string)
+    {
+      subarg_string_t* t = dynamic_cast<subarg_string_t*>(it);
+      if(t->val.find(c) != std::string::npos)
+        return true;
+    }
+  }
+  return false;
+}
+
+bool possibly_expands(arg_t* in)
+{
+  for(auto it: in->sa)
+    if( (it->type == _obj::subarg_subshell || it->type == _obj::subarg_variable ) && it->quoted == false)
+      return true;
+  return false;
+}
+
+bool possibly_expands(arglist_t* in)
+{
+  for(auto it: in->args)
+    if(possibly_expands(it))
+      return true;
+  return false;
+}
+
+// ** CLASS EXTENSIONS ** //
+
+/// GETTERS ///
+
+// property getters
+
+bool cmd_t::has_var_assign()
+{
+  if(this->args == nullptr || this->args->size() == 0)
+  {
+    return this->var_assigns.size()>0;
+  }
+  return this->is_argvar();
+}
+
+size_t cmd_t::arglist_size()
 {
   if(args==nullptr)
-    args = new arglist;
-
-  args->push_back(in);
+    return 0;
+  else
+    return args->size();
+}
+
+// string getters
+
+bool arg_t::is_string()
+{
+  return sa.size() == 1 && sa[0]->type == _obj::subarg_string;
+}
+
+std::string arg_t::string()
+{
+  if(!this->is_string())
+    return "";
+  return dynamic_cast<subarg_string_t*>(sa[0])->val;
+}
+
+std::string arg_t::first_sa_string()
+{
+  if(sa.size() <=0 || sa[0]->type != _obj::subarg_string)
+    return "";
+  return dynamic_cast<subarg_string_t*>(sa[0])->val;
+}
+
+std::string arglist_t::first_arg_string()
+{
+  if(args.size()<=0 || args[0] == nullptr)
+    return "";
+  return args[0]->string();
+}
+
+bool arg_t::can_expand()
+{
+  for(auto it: sa)
+  {
+    if(it->type != _obj::subarg_string && !it->quoted)
+      return true;
+  }
+  return false;
+}
+
+bool arglist_t::can_expand()
+{
+  bool arg_expands=false;
+  for(auto it: args)
+  {
+    arg_expands = it->can_expand();
+    if(arg_expands)
+      return true;
+  }
+  return false;
+}
+
+std::vector<std::string> arglist_t::strargs(uint32_t start)
+{
+  std::vector<std::string> ret;
+  bool t=opt_minify;
+  opt_minify=true;
+  for(uint32_t i=start; i<args.size(); i++)
+  {
+    ret.push_back(args[i]->generate(0));
+  }
+  opt_minify=t;
+  return ret;
+}
+
+std::string const& cmd_t::arg_string(uint32_t n)
+{
+  if(args!=nullptr && args->args.size()>n && args->args[n]->sa.size() == 1 && args->args[n]->sa[0]->type == _obj::subarg_string)
+    return dynamic_cast<subarg_string_t*>(args->args[n]->sa[0])->val;
+  return cmd_t::empty_string;
+}
+
+// subobject getters
+
+block_t* condlist_t::first_block()
+{
+  if(pls.size() > 0 && pls[0]->cmds.size() > 0)
+    return (pls[0]->cmds[0]);
+  else
+    return nullptr;
+}
+
+cmd_t* condlist_t::first_cmd()
+{
+  if(pls.size() > 0 && pls[0]->cmds.size() > 0 && pls[0]->cmds[0]->type == _obj::block_cmd)
+    return dynamic_cast<cmd_t*>(pls[0]->cmds[0]);
+  else
+    return nullptr;
+}
+
+cmd_t* brace_t::single_cmd()
+{
+  if( lst->cls.size() == 1 && // only one condlist
+      lst->cls[0]->pls.size() == 1 && // only one pipeline
+      lst->cls[0]->pls[0]->cmds.size() == 1 && // only one block
+      lst->cls[0]->pls[0]->cmds[0]->type == _obj::block_cmd) // block is a command
+    return dynamic_cast<cmd_t*>(lst->cls[0]->pls[0]->cmds[0]); // return command
+  return nullptr;
+}
+
+cmd_t* subshell_t::single_cmd()
+{
+  if( lst->cls.size() == 1 && // only one condlist
+      lst->cls[0]->pls.size() == 1 && // only one pipeline
+      lst->cls[0]->pls[0]->cmds.size() == 1 && // only one block
+      lst->cls[0]->pls[0]->cmds[0]->type == _obj::block_cmd) // block is a command
+    return dynamic_cast<cmd_t*>(lst->cls[0]->pls[0]->cmds[0]); // return command
+  return nullptr;
+}
+
+cmd_t* block_t::single_cmd()
+{
+  if(this->type == _obj::block_subshell)
+    return dynamic_cast<subshell_t*>(this)->single_cmd();
+  if(this->type == _obj::block_brace)
+    return dynamic_cast<brace_t*>(this)->single_cmd();
+  return nullptr;
+}
+
+/// MODIFIERS ///
+
+// simple setters
+
+void arg_t::set(std::string const& str)
+{
+  for(auto it: sa)
+    delete it;
+  sa.resize(0);
+  sa.push_back(new subarg_string_t(str));
+}
+
+void condlist_t::prune_first_cmd()
+{
+  if(pls.size()>0 && pls[0]->cmds.size()>0)
+  {
+    delete pls[0]->cmds[0];
+    pls[0]->cmds.erase(pls[0]->cmds.begin());
+  }
+}
+
+// add/extend
+
+void arg_t::insert(uint32_t i, std::string const& in)
+{
+  if(i>0 && i<=sa.size() && sa[i-1]->type == _obj::subarg_string)
+  {
+    subarg_string_t* t = dynamic_cast<subarg_string_t*>(sa[i-1]);
+    t->val += in;
+  }
+  else if(i<sa.size() && sa[i]->type == _obj::subarg_string)
+  {
+    subarg_string_t* t = dynamic_cast<subarg_string_t*>(sa[i]);
+    t->val = in + t->val;
+  }
+  else
+    sa.insert(sa.begin()+i, new subarg_string_t(in));
+}
+void arg_t::add(std::string const& in)
+{
+  this->insert(this->size(), in);
+}
+
+void arg_t::insert(uint32_t i, subarg_t* val)
+{
+  if(val->type == _obj::subarg_string)
+  {
+    subarg_string_t* tval = dynamic_cast<subarg_string_t*>(val);
+    if(i>0 && i<=sa.size() && sa[i-1]->type == _obj::subarg_string)
+    {
+      subarg_string_t* t = dynamic_cast<subarg_string_t*>(sa[i-1]);
+      t->val += tval->val;
+      delete val;
+    }
+    else if(i<sa.size() && sa[i]->type == _obj::subarg_string)
+    {
+      subarg_string_t* t = dynamic_cast<subarg_string_t*>(sa[i]);
+      t->val = tval->val + t->val;
+      delete val;
+    }
+    else
+      sa.insert(sa.begin()+i, val);
+  }
+  else
+    sa.insert(sa.begin()+i, val);
+}
+void arg_t::insert(uint32_t i, arg_t const& a)
+{
+  sa.insert(sa.begin()+i, a.sa.begin(), a.sa.end());
+}
+
+void arglist_t::insert(uint32_t i, arg_t* val)
+{
+  args.insert(args.begin()+i, val);
+}
+void arglist_t::insert(uint32_t i, arglist_t const& lst)
+{
+  args.insert(args.begin()+i, lst.args.begin(), lst.args.end());
+}
+
+void cmd_t::add(arg_t* in)
+{
+  if(args==nullptr)
+    args = new arglist_t;
+
+  args->add(in);
+}
+
+void condlist_t::add(pipeline_t* pl, bool or_op)
+{
+  if(pls.size() > 0)
+    or_ops.push_back(or_op);
+  pls.push_back(pl);
+}
+
+void list_t::insert(uint32_t i, condlist_t* val)
+{
+  if(i<0)
+    cls.insert(cls.end(), val);
+  else
+    cls.insert(cls.begin()+i, val);
+}
+void list_t::insert(uint32_t i, list_t const& lst)
+{
+  if(i<0)
+    cls.insert(cls.end(), lst.cls.begin(), lst.cls.end());
+  else
+    cls.insert(cls.begin()+i, lst.cls.begin(), lst.cls.end());
+}
+
+void shmain::concat(shmain* in)
+{
+  lst->insert(lst->size(), *in->lst);
+  in->lst->cls.resize(0);
+  if(this->shebang == "")
+    this->shebang = in->shebang;
+}
+
+// special modifiers
+
+void condlist_t::negate()
+{
+  // invert commands
+  for(uint32_t i=0; i<pls.size(); i++)
+    pls[i]->negated = !pls[i]->negated;
+  // invert bool operators
+  for(uint32_t i=0; i<or_ops.size(); i++)
+    or_ops[i] = !or_ops[i];
 }

src/util.cpp

@@ -7,8 +7,10 @@
 #include <tuple>
 
 #include <iostream>
+#include <fstream>
 
 #include <ztd/shell.hpp>
+#include <ztd/color.hpp>
 
 std::string indenting_string="\t";
 
@@ -20,15 +22,20 @@ std::string indent(int n)
   return ret;
 }
 
-std::string basename(std::string const& in)
+std::string cut_last(std::string const& in, char c)
 {
-  size_t slr=in.rfind('/');
+  size_t slr=in.rfind(c);
   if(slr != std::string::npos)
     return in.substr(slr+1);
   else
     return in;
 }
 
+std::string basename(std::string const& in)
+{
+  return cut_last(cut_last(in, '/'), ' ');
+}
+
 std::string dirname(std::string const& in)
 {
   size_t slr=in.rfind('/');
@@ -189,6 +196,19 @@ std::string stringReplace(std::string subject, const std::string& search, const
   return subject;
 }
 
+std::string escape_chars(std::string subject, const char* chars)
+{
+  for(size_t i=0; i<subject.size(); i++)
+  {
+    if(is_in(subject[i], chars))
+    {
+      subject.insert(subject.begin()+i, '\\');
+      i++;
+    }
+  }
+  return subject;
+}
+
 std::string repeatString(std::string const& str, uint32_t n)
 {
   std::string ret;
@@ -197,16 +217,15 @@ std::string repeatString(std::string const& str, uint32_t n)
   return ret;
 }
 
-void printFormatError(ztd::format_error const& e, bool print_line)
+void printFormatError(format_error const& e, bool print_line)
 {
-  printErrorIndex(e.data(), e.where(), e.what(), e.origin(), print_line);
-}
+  const char* in = e.data();
 
-void printErrorIndex(const char* in, const int index, const std::string& message, const std::string& origin, bool print_line)
-{
-  int i=0, j=0; // j: last newline
-  int line=1; //n: line #
-  int in_size=strlen(in);
+  uint64_t index = e.where();
+
+  uint64_t i=0, j=0; // j: last newline
+  uint64_t line=1; //n: line #
+  uint64_t in_size=strlen(in);
   if(index >= 0)
   {
     while(i < in_size && i < index)
@@ -219,59 +238,25 @@ void printErrorIndex(const char* in, const int index, const std::string& message
       i++;
     }
     while(i < in_size && in[i]!='\n')
-    {
       i++;
-    }
   }
-  if(origin != "")
+  std::cerr << ztd::color::b_white;
+  fprintf(stderr, "%s:%lu:%lu: ", e.origin(), line, index-j+1);
+
+  ztd::color level_color;
+  const std::string& level = e.level();
+  if(level == "error")
+    level_color = ztd::color::b_red;
+  else if(level == "warning")
+    level_color = ztd::color::b_magenta;
+  else if(level == "info")
+    level_color = ztd::color::b_cyan;
+
+  std::cerr << level_color << e.level() << ztd::color::none;
+  fprintf(stderr, ": %s\n", e.what());
+  if(print_line)
   {
-    fprintf(stderr, "%s:%u:%u: %s\n", origin.c_str(), line, index-j+1, message.c_str());
-    if(print_line)
-    {
-      std::cerr << std::string(in+j, i-j) << std::endl;
-      std::cerr << repeatString(" ", index-j) << '^' << std::endl;
-    }
+    std::cerr << std::string(in+j, i-j) << std::endl;
+    std::cerr << repeatString(" ", index-j) << '^' << std::endl;
   }
 }
-
-
-int execute(shmain* sh, std::vector<std::string>& args)
-{
-  std::string data=sh->generate();
-
-  std::string filename = basename(args[0]);
-
-  // generate path
-  std::string tmpdir = (getenv("TMPDIR") != NULL) ? getenv("TMPDIR") : "/tmp" ;
-  std::string dirpath = tmpdir + "/lxsh_" + ztd::sh("tr -dc '[:alnum:]' < /dev/urandom | head -c10");
-  std::string filepath = dirpath+'/'+filename;
-
-  // create dir
-  if(ztd::exec("mkdir", "-p", dirpath).second)
-  {
-    throw std::runtime_error("Failed to create directory '"+dirpath+'\'');
-  }
-
-  // create stream
-  std::ofstream stream(filepath);
-  if(!stream)
-  {
-    ztd::exec("rm", "-rf", dirpath);
-    throw std::runtime_error("Failed to write to '"+filepath+'\'');
-  }
-
-  // output
-  stream << data;
-  stream.close();
-  if(ztd::exec("chmod", "+x", filepath).second != 0)
-  {
-    ztd::exec("rm", "-rf", dirpath);
-    throw std::runtime_error("Failed to make '"+filepath+"' executable");
-  }
-
-  // exec
-  int retval=_exec(filepath, args);
-  ztd::exec("rm", "-rf", dirpath);
-
-  return retval;
-}

test/a.bash

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+diff <(echo a) <(echo b)
+
+write_to_file() { echo "$2" > "$1"; }
+write_to_file >(grep bar) bar
+wait $!
+
+echo a &> /tmp/foo
+echo b >& /tmp/bar
+echo c &>> /tmp/foo
+
+cat /tmp/bar /tmp/foo
+rm /tmp/bar /tmp/foo
+
+
+TOTO="foo
+bar"
+grep ar <<< ar$TOTO
+
+declare -a A
+A=("fo o" bar)
+echo ${A[1]}
+
+declare -A B
+B[foo]=ta
+B[bar]=tu
+echo ${B[foo]}
+echo ${B[bar]}
+echo ${B[*]}
+
+C=([foo]=bar [bar]=foo)
+echo ${C[foo]}
+echo ${C[bar]}
+echo ${C[*]}
+
+BAR=FOO
+echo ${!BAR}
+
+[[ $DEBUG == true ]] && echo debug
+
+a=a
+[[ $a = a && foo = fo* && bar =~ b.r || 2 < 3 ]]
+
+for I in A B C ; do
+  echo "$I"
+done > >(cat)

test/arithmetic.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo $(( (2+1)*3 ))
+echo $(( $((2+1))*3 ))

test/array.bash

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+TOTO=(toto tata)
+TOTO[0]=titi
+TOTO[1]+=tu
+echo ${TOTO[0]}
+echo ${TOTO[1]}
+echo ${TOTO[*]}
+
+TOTO+=(2)
+echo $((TOTO[2]+1))
+echo $((${TOTO[2]}+2))
+
+declare -a TUTU
+TUTU=(titi "tu tu")
+echo ${TUTU[0]}
+echo ${TUTU[1]}
+echo ${TUTU[*]}
+echo "${TUTU[*]}"
+
+declare -A A
+A[to]=ta
+A[ti]=tu
+echo ${A[to]}
+echo ${A[ti]}
+echo ${A[*]}
+
+declare -A B
+B=([to]=ta [ti]=tu)
+echo ${B[to]}
+echo ${B[ti]}
+echo ${B[*]}
+echo "${B[*]}"
+
+toto=tata
+C=()
+C+=($toto)
+echo ${C[@]}

test/backtick.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+echo $(echo toto)
+echo $(printf %s\\n tata)
+echo `echo titi`
+echo `printf '%s\\n' tutu`
+

test/brace.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+{ echo tata ; echo a; } | sed 's|a|toto|g'
+
+echo a | { grep a && echo b; }
+
+{ { echo tata ; } }

test/braceexp.bash

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+echo tot{a,o,i}
+echo {1..10}
+echo {0..10..2}
+echo {1..10..2}
+echo tot{a,o,i}{1..3}tata
+echo :{{a..z},{A..Z}}

test/case.sh

@@ -0,0 +1,17 @@
+
+echo "$*" | case $1 in
+  to*) echo toto ;;
+  tata) echo wew tata ;;
+  a | b)
+    echo titi
+    ;;
+  *)
+    cat ;;
+esac
+
+case foo in bar)echo a;;foo)echo b;esac
+case foo in bar)echo a;;foo)echo b
+esac
+
+case far in foo) echo a;;bar)
+esac

test/comment.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo toto#
+echo toto#tata
+echo toto #tata

test/complex.sh

@@ -0,0 +1,22 @@
+while [ -z "$I" ]
+do
+case $U in
+*)echo toto;I=y
+esac
+done
+echo "$I"
+
+case toto in
+  tutu) ;;
+
+  toto)
+cat << EOF
+toto
+EOF
+;;
+  tata)
+esac
+
+echo to 2>&1
+echo to2 >&1
+echo to$(echo 2) >&1

test/debashify.bash

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+readonly tutu titi=tata
+echo "$tutu $titi"
+
+diff <(echo a) <(echo b)
+
+write_to_file() { echo "$2" > "$1"; }
+write_to_file >(grep tutu) tutu
+wait $!
+
+echo a &> /tmp/toto
+echo b >& /tmp/tata
+echo c &>> /tmp/toto
+
+cat /tmp/tata /tmp/toto
+rm /tmp/tata /tmp/toto
+
+
+TOTO="ta
+to"
+grep ta <<< toto$TOTO
+
+TATA=ti
+TATA+=tu
+echo $TATA
+
+[[ $DEBUG == true ]] && echo debug
+
+[ $((RANDOM+RANDOM)) -gt 0 ]
+echo randomstat: $?
+
+a=a
+[[ $a = a && foo = fo* && bar =~ b.r || 2 < 3 ]]
+echo $?
+
+N=1
+TOTO=tatitu
+echo "${TOTO:2}"
+echo "${TOTO:$N:2}"
+
+echo ${TOTO:-tutu}
+echo ${TITI:-bar}
+
+TATA=TOTO
+echo ${!TATA}
+
+for I in A B C ; do
+  echo "$I"
+done 2>&1 > >(grep A)

test/dequote.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+
+
+toto=tutu
+tata=titi
+
+echo "toto tata titi"
+echo "toto"
+
+echo "tata"titi
+
+echo "ta ta"
+
+echo "$toto"tata
+
+echo "$toto"
+
+echo $toto"tata"
+
+echo $"toto"
+
+toto="$toto"
+
+toto="$toto"tata
+echo "$toto"
+toto="tata"$tata
+echo "$toto"
+toto=$toto"tata"
+echo "$toto"
+toto="$toto".tata
+echo "$toto"
+
+tata="ta ta"
+echo "$tata"
+
+echo "$"
+echo \$
+
+toto=tutu
+tata=titi
+
+echo 'toto tata titi'
+echo 'toto'
+
+echo 'tata'titi
+
+echo 'ta ta'
+
+echo '$toto'tata
+
+echo '$toto'
+
+echo $toto'tata'
+
+echo $'toto'
+
+toto='$toto'
+
+toto='$toto'tata
+echo '$toto'
+toto='tata'$tata
+echo '$toto'
+toto=$toto'tata'
+echo '$toto'
+toto='$toto'.tata
+echo '$toto'
+
+tata='ta ta'
+echo '$tata'
+
+echo '$'
+
+
+cat << EOF
+"toto"
+EOF
+
+printf "%s\n" "" ""'' "toto"

test/echo.bash

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+echo -n titi:
+echo -e 'tata\n'
+echo -E 'tutu\n'
+
+echo -n tata tutu tete
+
+toto="to  to"
+echo $toto
+
+echo to $toto

test/err/brace.sh

@@ -0,0 +1 @@
+{ ; }

test/err/err.bash

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+var[a
+read var+=a
+export var+=a
+export var=()
+
+[[ a = b ]] toto
+
+echo >() <()
+
+function toto-titi{ true; }

test/err/err.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+
+read var=a
+var+=a
+var=(foo)
+
+$((var~2))
+
+${!var}
+${~}
+${#var-a}
+`echo \`echo\`  `
+
+$(( (var) )
+
+>& /dev/null
+echo &> /dev/null
+
+cat 2< file
+cat <<< var
+echo >
+
+echo &| cat
+echo |& cat
+
+[[ a = b ]] foo
+
+()
+
+fct() abc
+{ ; }
+
+fct() { }
+
+typeset var
+var=val read var
+case foo ; esac
+case foo in aiae ; esac
+case foo in ) ; esac
+case foo in a) ; b) esac
+
+for 2 in a ; do true ; done
+for foo do ; do true ; done
+for foo & ; do true ; done
+for I in ; true ; done
+
+while
+do true ; done
+
+while true ; do
+done
+
+if true ;then
+fi
+
+if
+then true ; fi
+
+if true ; then true ; else
+fi
+
+fct-foo() { true; }
+
+function foo { true; }
+
+{ foo; } bar

test/fct.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+toto() {
+  echo toto
+}
+
+tata () {
+  echo tata
+}
+
+toto

test/for.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+__for_fct() {
+  for I in ; do
+    echo $I
+  done
+}
+
+for N
+do
+  echo $N
+done
+
+for I in $(seq 1 10); do
+  echo "toto $I"
+done
+
+__for_fct toto tata

test/heredocument.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+cat << EOF
+toto
+tata
+EOF
+
+toto=toto
+cat << EOF | grep toto
+$toto
+tata
+EOF
+
+cat << EOF
+'
+EOF
+
+cat << EOF |
+azjeha
+kijejaze
+ljksdjk
+EOF
+cut -c1
+
+grep -q toto << EOF &&
+toto
+EOF
+echo found toto
+
+{ cat << EOF | grep toto; }
+toto
+tata
+EOF
+
+( cat << EOF | grep toto )
+toto
+tata
+EOF
+
+{ cat << EOF | grep toto && echo true; echo eyy; }
+toto
+tata
+EOF
+
+( cat << EOF | grep toto && echo true ; echo eyy )
+toto
+tata
+EOF

test/if.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ -n "$DEBUG" ]
+then
+  echo "set"
+elif [ -n "$TOTO" ]
+then
+  echo "toto lol"
+else
+  echo "not set"
+fi

test/include.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+RAW="$(%include -f pipe.sh brace.sh)"
+
+echo "$RAW"
+
+{ %include -f pipe.sh; } | grep -q arch && echo "btw i use arch"
+
+%include *.sh

test/manip.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+var="marijuana"
+echo ${#var}
+
+echo ${var-foo}
+echo ${var+foo}
+echo ${foo-foo}
+echo ${foo+foo}
+
+echo ${var#*a}
+echo ${var##*a}
+echo ${var%a*}
+echo ${var%%a*}
+

test/pipe.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+grep "^ID=" /etc/os-release | cut -d '=' -f2-
+
+echo toto | #
+grep to
+
+echo '#toto' | grep '#toto'

test/prompt.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+read -r prompt || echo $?

test/redir.sh

@@ -0,0 +1,5 @@
+{ echo toto >&2; } 2>&1
+
+{ echo tata; }>/dev/null
+
+grep abc < test/redir.sh

test/resolve.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+echo "a$(%resolve echo tata titi)b"
+
+echo $(%resolve cut -d ' ' -f1 /proc/uptime)s
+
+%resolve echo "TIME=$(cut -d ' ' -f1 /proc/uptime)s;echo This was compiled at \${TIME}s uptime"
+
+FOO=$(%resolve echo bar) echo foo

test/subshell.sh

@@ -0,0 +1,10 @@
+TOTO=toto
+(TOTO=tata; echo $TOTO; echo a) | sed 's|a|titi|g'
+echo $TOTO
+
+echo a | ( grep a && echo b )
+
+echo ab | ( grep a )
+pwd
+(cd /)
+pwd

test/var.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+foo()
+{
+  echo $FOO
+}
+
+TOTO=tata
+TATA=titi
+echo $TOTO
+echo "$TOTO  $TATA$TITI"
+echo "${AYE-aye}"
+
+export TUTU=ta
+
+foo
+FOO=bar foo
+BAR=foo foo
+ABCD=$(FOO=true foo)
+echo $ABCD
+nul=/dev/null
+echo toto > "$nul"
+
+somevar=val
+echo $somevar
+unset somevar
+echo $somevar
+
+cat << EOF
+$TOTO
+EOF

test/while.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+I=0
+while [ $I -lt 10 ]
+do
+  I=$((I+1))
+  echo "$I"
+done
+
+I=0
+until [ $I -eq 10 ]
+do
+  I=$((I+1))
+  echo "$I"
+done
+
+I=0
+while
+  I=$((I+1))
+  echo "$I"
+  [ $I -lt 10 ]
+do true
+done