Merge branch 'epsilon' into patch-1

README.md

@@ -205,6 +205,7 @@ To learn more checkout how to apply meander [strategies](doc/strategies.md)
 * [Meander: The answer to map fatigue](http://timothypratley.blogspot.com/2019/01/meander-answer-to-map-fatigue.html)
 * [Meander for Practical Data Transformation](https://jimmyhmiller.github.io/meander-practical)
 * [Introduction to Term Rewriting with Meander](https://jimmyhmiller.github.io/meander-rewriting)
+* [Building Meander in Meander](https://jimmyhmiller.github.io/building-meander-in-meander)
 
 ### Talks
 

doc/cookbook.md

@@ -1,5 +1,279 @@
 Please add your own tips and tricks! You can edit this file from Github by clicking then pencil icon in the top right of the file view.
 
+---
+
+
+## Common Patterns
+
+### Capture Variable From Pattern Match
+- How to capture a variable from a pattern match? Use `:as`
+    ```clojure
+    (def a_opprmdefblk [(ophirPrmDef :inBoneTrk ctidChannel #{:EArg-In})
+                        (ophirPrmDef :inoutBoneTrk ctidChannel #{:EArg-In :EArg-Out})
+                        (ophirPrmDef :outBoneTrk ctidChannel #{:EArg-Out})])
+    (m/search
+     a_opprmdefblk
+     [(m/or {:argFlags #{:EArg-Out} :as !argOut}
+            {:argFlags #{:EArg-In} :as !argIn})
+      ...]
+     {:opmirArgIn !argIn
+      :opmirArgOut !argOut})
+    ```
+
+
+### Sequence Transformation
+
+- Desired Result
+  ```clojure
+  ;EBNF
+  ns <= "obj" | "oppas" | "dc"
+  segattr <= ["/"] "@" alphanumeric
+  segobj  <= ["/"] alphanumeric
+  xpath   <= ns (segattr|segobj) {(segattr|segobj)}
+
+  ; Input
+  "obj:/myobj/mychild/@myattrib"
+  ;; Result =>
+  {:ns :obj,
+  :xsegs
+  ({:segkind :seg-chld, :segpath ""}
+    {:segkind :seg-chld, :segpath "myobj"}
+    {:segkind :seg-chld, :segpath "mychild"}
+    {:segkind :seg-attr, :segpath "@myattrib"})}
+  ```
+
+- What this shows:
+  - Input: a tokenized string
+  - Make sure tokens match order pattern (`nstoken xseg {xseg}` )
+  - Transform each of the tokens based on the token
+    ```clojure
+    nstoken =>
+      case "obj": :objstore
+      default: (keyword nstoken)
+    ```
+
+- **Normal Clojure**
+  ```clojure
+  (defn initOppath-clj [axpath]
+      (let [nsandpath (str/split axpath #"[:]" 2)
+            nsstr (first nsandpath)
+            pathtokens (->
+                        nsandpath
+                        (nth 1)
+                        (str/split #"[/]"))]
+        {:ns (case nsstr
+               "op"    :op
+               "obj"   :obj
+               "oppas" :oppas)
+         :xsegs (map
+                 #(if (= (first %1) \@)
+                    (->OppathSeg :seg-attr %1)
+                    (->OppathSeg :seg-chld %1))
+                 pathtokens)}))
+  ```
+
+- **Meander**   
+    - **Naive attempt**: 
+        ```clojure
+        (defn initOppath-m1 [axpath]
+          (let [axptokens (str/split axpath #"[/:]")]
+            {:ns (m/match (first axptokens)
+                   (m/and ?ns (m/or "op" "obj" "oppas"))
+                   (keyword ?ns))
+             :xsegs (map
+                     #(if (= (first %1) \@)
+                        (->OppathSeg :seg-attr %1)
+                        (->OppathSeg :seg-chld %1))
+                     (rest axptokens))}))
+        ```
+
+    - **Second Attempt:** Better but a nitpick is the functional transformation is on the pattern matching clause where conceptually feels like it should go in the generation part
+        ```clojure
+        (defn initOppath-m2 [axpath]
+              (m/match (str/split axpath #"[/:]")
+                (m/with [%segattr (m/pred #(= (first %1) \@)    (m/app #(->OppathSeg :seg-attr %1) !seg))
+                         %segobj  (m/pred #(not= (first %1) \@) (m/app #(->OppathSeg :seg-chld %1) !seg))]
+                        [(m/re #"obj|oppas|dc" ?ns)
+                         . (m/or %segobj %segattr) ...])
+                {:ns (keyword ?ns) :xsegs !seg}))        
+        ```
+
+    - **Cleaner Solution** Use a helper to construct the xseg:
+
+        ```clojure
+        (defn make-xseg [val]
+          (m/rewrite val
+            (m/re #"@.*" ?val)
+            {:kind :seg-attr :val ?val}
+
+            (m/re #"[^@].*" ?val)
+            {:kind :seg-chld :val ?val}
+
+            ?val
+            {:kind :unknown :val ?val}))
+
+
+        (m/rewrite ["oppas" "obj1" "@attr1" "@attr2" "obj2"]
+          [(m/re #"obj|oppas|dc" ?ns) . !segs ...]
+          {:ns (m/keyword ?ns)
+           :xsegs [(m/app make-xseg !segs) ...]})
+        ;; =>
+        {:ns :oppas,
+         :xsegs
+         [{:kind :seg-chld, :val "obj1"}
+          {:kind :seg-attr, :val "@attr1"}
+          {:kind :seg-attr, :val "@attr2"}
+          {:kind :seg-chld, :val "obj2"}]}
+        ```
+
+    - **Concise Using Recursion**: The second uses `m/cata` on the left or right side:
+        - Left side
+            ```clojure
+            (m/rewrite ["oppas" "obj1" "@attr1" "@attr2" "obj2"]
+              [(m/re #"obj|oppas|dc" ?ns) . (m/cata !segs) ...]
+              {:ns (m/keyword ?ns)
+               :xsegs [!segs ...]}
+
+              (m/re #"@.*" ?val)
+              {:kind :seg-attr :val ?val}
+
+              (m/re #"[^@].*" ?val)
+              {:kind :seg-chld :val ?val}
+
+              ?val
+              {:kind :unknown :val ?val})
+            ```
+
+        - Right side
+            ```clojure
+            (m/rewrite ["oppas" "obj1" "@attr1" "@attr2" "obj2"]
+              [(m/re #"obj|oppas|dc" ?ns) . !segs ...]
+              {:ns (m/keyword ?ns)
+               :xsegs [(m/cata !segs) ...]}
+
+              (m/re #"@.*" ?val)
+              {:kind :seg-attr :val ?val}
+
+              (m/re #"[^@].*" ?val)
+              {:kind :seg-chld :val ?val}
+
+              ?val
+              {:kind :unknown :val ?val})
+            ```
+
+    - **Final Solution:** Cata on the right side can be used to construct a value to be recursively rewritten. It’s the dual of the left.
+        ```clojure
+        (m/rewrite ["oppas" "obj1" "@attr1" "@attr2" "obj2"]
+          [(m/re #"obj|oppas|dc" ?ns) . !segs ...]
+          {:ns (m/keyword ?ns)
+           :xsegs [(m/cata ($EXAMPLE !segs)) ...]}
+
+          ($EXAMPLE (m/re #"@.*" ?val))
+          {:kind :seg-attr :val ?val}
+
+          ($EXAMPLE (m/re #"[^@].*" ?val))
+
+          {:kind :seg-chld :val ?val}
+
+          ($EXAMPLE ?val)
+          {:kind :unknown :val ?val})
+        ;; =>
+        {:ns :oppas,
+         :xsegs
+         [{:kind :seg-chld, :val "obj1"}
+          {:kind :seg-attr, :val "@attr1"}
+          {:kind :seg-attr, :val "@attr2"}
+          {:kind :seg-chld, :val "obj2"}]}
+        ```
+
+### Split stream based on filter and project   (1-to-many)
+- Pseudo code:
+    ```clojure
+    filter(
+      (predA? x) => (projA x) :as !projAseq
+      (predB? x) => (projB x) :as !projBseq
+    )
+    ```
+
+- Clojure Code
+    ```clojure        
+    ;; Test Data
+    (def arglist [{:name :inBoneTrk    :argFlags #{:EArg-In}}
+                  {:name :inoutBoneTrk :argFlags #{:EArg-In :EArg-Out}}
+                  {:name :outBoneTrk   :argFlags #{:EArg-Out}}])
+    ;; Using match
+    (m/match
+     arglist
+      [(m/or {:argFlags #{:EArg-Out} :as !argOut}
+             {:argFlags #{:EArg-In} :as !argIn})
+       ...]
+      {:opmirArgIn !argIn
+       :opmirArgOut !argOut})
+    ;; =>
+    {:opmirArgIn  [{:name     :inBoneTrk
+                    :argFlags #{:EArg-In}}]
+     :opmirArgOut [{:name     :inoutBoneTrk
+                    :argFlags #{:EArg-Out :EArg-In}} 
+                   {:name     :outBoneTrk
+                    :argFlags #{:EArg-Out}}]}
+
+    ```
+
+- Now let's use m/search to see the difference
+    ```clojure
+    (m/search
+     arglist
+     [(m/or {:argFlags #{:EArg-Out} :as !argOut}
+            {:argFlags #{:EArg-In} :as !argIn})
+      ...]
+     {:opmirArgIn !argIn
+      :opmirArgOut !argOut})
+    ;; =>
+    ({:opmirArgIn [{:name :inBoneTrk, :argFlags #{:EArg-In}}]
+      :opmirArgOut [{:name :inoutBoneTrk, :argFlags #{:EArg-Out :EArg-In}} 
+                    {:name :outBoneTrk, :argFlags #{:EArg-Out}}]}
+     {:opmirArgIn [{:name :inBoneTrk, :argFlags #{:EArg-In}} 
+                   {:name :inoutBoneTrk, :argFlags #{:EArg-Out :EArg-In}}]
+      :opmirArgOut [{:name :outBoneTrk, :argFlags #{:EArg-Out}}]})
+    ```
+
+- Now let's look using m/scan
+    ```clojure
+    (m/search
+     arglist
+     (m/scan {:argFlags #{:EArg-In} :as ?argIn})
+     ?argIn)
+    ;; =>
+    ({:name     :inBoneTrk
+      :argFlags #{:EArg-In}} 
+     {:name     :inoutBoneTrk
+      :argFlags #{:EArg-Out :EArg-In}})
+    ```
+
+- Now let's look at m/scan with a memory variable
+    ```clojure
+    (m/search
+     arglist
+     (m/scan {:argFlags #{:EArg-In} :as !argIn})
+     !argIn)
+    ;; =>
+    ([{:name     :inBoneTrk
+       :argFlags #{:EArg-In}}]
+     [{:name     :inoutBoneTrk
+       :argFlags #{:EArg-Out :EArg-In}}])
+    ```
+
+---
+
+## TODO  
+- How to do EBNF like production rules.  Ex: 
+    ```clojure
+    token ::= (:arg-in|:arg-out) ?argname
+    pseudocode-result:: (str (emit-in ?arg-attr)|emit-out :arg-attr) ?argname)    
+    ```
+
+---
+
 ## Reuse subpatterns in other patterns
 
 ```clojure

src/meander/match/epsilon.cljc

@@ -1848,7 +1848,7 @@
           ;; expr to eliminate clauses that won't match which can
           ;; be too aggressive when a catamorphism is present.
           (binding [*cata-symbol* (get search-analysis :cata-symbol)]
-            (let [cata-return (gensym "CATA_RETURN__")
+            (let [cata-return (gensym "R__")
                   ir {:op :def
                       :symbol *cata-symbol*
                       :target-arg target
@@ -1960,7 +1960,7 @@
              matrix (if final-clause
                       (into [] (take-while (partial not= final-clause)) matrix)
                       matrix)]
-         {:cata-symbol (gensym "CATA__FN__")
+         {:cata-symbol (gensym "C__")
           :contains-cata? contains-cata?
           :errors errors
           :expr (get result :expr)
@@ -1985,7 +1985,7 @@
                      (gensym "TARGET__"))]
         (if contains-cata?
           (binding [*cata-symbol* (get find-analysis :cata-symbol)]
-            (let [cata-return (gensym "CATA_RETURN__")
+            (let [cata-return (gensym "R__")
                   ir {:op :def
                       :symbol *cata-symbol*
                       :target-arg target

src/meander/match/ir/epsilon.cljc

@@ -929,7 +929,6 @@ compilation decisions."
 (defn infer-type-seq
   {:private true}
   [env node form]
-
   (cond
     ;; If it is quoted then a seq can't be a function call so it is a seq
     (and (r.util/quoted? form)
@@ -942,7 +941,8 @@ compilation decisions."
 
     ;; if it is a list, it is a seq
     (and (seq? form)
-         (= (r.util/expand-symbol (first form) *env*) 'clojure.core/list))
+         (and (symbol? (first form))
+              (= (r.util/expand-symbol (first form) *env*) 'clojure.core/list)))
     (add-type-to-env env (:symbol node) SeqInterface)
 
     ;; otherwise just check the type
@@ -1480,7 +1480,7 @@ compilation decisions."
     `(let [~R__1 ~body-1-code]
        (if (r.match.runtime/fail? ~R__1)
          ~body-2-code
-         r.match.runtime/FAIL))))
+         ~fail))))
 
 (defmethod compile* :pass
   [ir fail kind]

src/meander/match/syntax/epsilon.cljc

@@ -465,15 +465,22 @@
   ([node]
    (expand-ast node r.environment/default))
   ([node env]
+   (-> (expand-ast-top-down node env)
+       (expand-ast-bottom-up env)
+       (r.syntax/rename-refs)
+       (r.syntax/consolidate-with))
+   #_
    (let [node* (-> (expand-ast-top-down node env)
                    (expand-ast-bottom-up env)
                    (r.syntax/rename-refs)
                    (r.syntax/consolidate-with))
+         ;; Below introduced in
+         ;; 68cd2dd900aa70e913a4fa4119f6b63ff5de6c37
+         ;; causes problems with compilation _ pattern compilation.
          node* (if (seq (get node* :bindings))
                  node*
                  (get node* :body))]
      node*)))
-
 ;; ---------------------------------------------------------------------
 ;; Syntax analysis
 

src/meander/substitute/epsilon.cljc

@@ -151,10 +151,11 @@
     {:argument ?argument :as ?node}
     (if-some [cata-symbol (get env :cata-symbol)]
       (let [[argument env] (compile* ?argument env)
-            form `(let [CATA_RESULT# (~cata-symbol ~argument)]
-                    (if (r.match.runtime/fail? CATA_RESULT#)
+            result (gensym "R__")
+            form `(let [~result (~cata-symbol ~argument)]
+                    (if (r.match.runtime/fail? ~result)
                       (throw r.subst.runtime/FAIL)
-                      (nth CATA_RESULT# 0)))]
+                      (nth ~result 0)))]
         [form env])
       (let [env (update env :data conj {:error :cata-not-bound})]
         [`(throw (ex-info "cata not bound" {})) env]))))