diff --git a/Makefile b/Makefile index 5829c583..704e1b4e 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=externals -PYTHON ?= python +PYTHON ?= pypy PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy @@ -22,10 +22,16 @@ build_with_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target.py make compile_basics +build_with_jit2: fetch_externals + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target2.py + build_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target.py make compile_basics +build_no_jit2: fetch_externals + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target2.py + compile_basics: @echo -e "\n\n\n\nWARNING: Compiling core libs. If you want to modify one of these files delete the .pxic files first\n\n\n\n" ./pixie-vm -c pixie/uv.pxi -c pixie/io.pxi -c pixie/stacklets.pxi -c pixie/stdlib.pxi -c pixie/repl.pxi @@ -60,7 +66,10 @@ run: run_interactive: - @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py + @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py pixie/compiler.pxi + +run_interactive2: + @PYTHONPATH=$(PYTHONPATH) pypy target2.py run_interactive_stacklets: @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py pixie/stacklets.pxi @@ -87,3 +96,8 @@ clean: clean_pxic rm -rf ./externals rm -f ./pixie-vm rm -f ./*.pyc + +compile_rpython: + touch ./bootstrap.pxic + rm ./bootstrap.pxic + ../pixie-old/pixie-vm -l . pixie/compile-bootstrap.pxi diff --git a/pixie/ast-output.pxi b/pixie/ast-output.pxi new file mode 100644 index 00000000..9dda8a8e --- /dev/null +++ b/pixie/ast-output.pxi @@ -0,0 +1,124 @@ +(ns pixie.ast-output + (:require [pixie.ast.internal :as iast] + [pixie.ast :as ast])) + +(defprotocol ToNativeAST + (-to-ast [this])) + +(defn meta-ast [ast] + (let [{:keys [line file line-number column-number] :as m} (or (meta (:form ast)) + (:meta (:env ast))) + line (if (string? line) + line + (apply str @line))] + + (iast/->Meta (or line + "") + (or file + "") + (or line-number -1) + (or column-number 1)))) + +(defn to-ast [this] + (-to-ast (simplify this))) + +(extend-protocol ToNativeAST + + ast/If + (-to-ast [{:keys [test then else] :as ast}] + (iast/->If (to-ast test) + (to-ast then) + (to-ast else) + (meta-ast ast))) + + ast/Let + (-to-ast [{:keys [bindings body] :as ast}] + (iast/->Let (apply array (map #(keyword (name (:name %))) + bindings)) + (apply array (map #(to-ast (:value %)) + bindings)) + (to-ast body) + (meta-ast ast))) + ast/Do + (-to-ast [{:keys [statements ret] :as ast}] + (let [args-array (make-array (inc (count statements)))] + (dotimes [idx (count statements)] + (aset args-array idx + (to-ast (nth statements idx)))) + + (aset args-array (count statements) + (to-ast ret)) + + (iast/->Do args-array + (meta-ast ast)))) + + ast/LetBinding + (-to-ast + [{:keys [name] :as ast}] + (iast/->Lookup (keyword (pixie.stdlib/name name)) + (meta-ast ast))) + + + ast/Const + (-to-ast [{:keys [form] :as ast}] + (iast/->Const form (meta-ast ast))) + + ast/Invoke + (-to-ast [{:keys [args env] :as ast}] + (let [args-array (make-array (count args))] + (dotimes [idx (count args)] + (aset args-array idx + (to-ast (nth args idx)))) + + (iast/->Invoke args-array + (meta-ast ast)))) + + ast/Recur + (-to-ast [{:keys [args env] :as ast}] + (let [args-array (make-array (count args))] + (dotimes [idx (count args)] + (aset args-array idx + (to-ast (nth args idx)))) + + (iast/->Recur args-array + (meta-ast ast)))) + + ast/Var + (-to-ast [{:keys [ns var-name] :as ast}] + (iast/->VDeref (name ns) + var-name + (meta-ast ast))) + + ast/VarConst + (-to-ast [{:keys [ns name env] :as ast}] + (iast/->VarConst ns + name + (meta-ast ast))) + + ast/FnBody + (-to-ast [{:keys [name args body] :as ast}] + (iast/->Fn (keyword (pixie.stdlib/name name)) + (to-array (map (fn [x] + (keyword (pixie.stdlib/name x))) + args)) + (to-ast body) + (meta-ast ast))) + + ast/Binding + (-to-ast [{:keys [name] :as ast}] + (iast/->Lookup (keyword (pixie.stdlib/name name)) + (meta-ast ast))) + + Object + (-to-ast [this] + (println "Encoding" this "of type" (type this)) + (throw [:pixie.stdlib/IllegalArgumentException + (str "Can't encode " this)]))) + +(defn simplify [ast] + (let [simplified (ast/simplify-ast ast)] + (if (identical? simplified ast) + ast + (simplify simplified)))) + + diff --git a/pixie/ast.pxi b/pixie/ast.pxi new file mode 100644 index 00000000..259cba2c --- /dev/null +++ b/pixie/ast.pxi @@ -0,0 +1,194 @@ +(ns pixie.ast) + +(defprotocol IAst + (children-keys [this])) + +(extend-type Object + IAst + (children-keys [this] + (throw [:pixie.stdlib/IllegalArgumentException + (str "Can't get AST keys of " this)]))) + +(defrecord Meta [line column-number]) +(defrecord LineMeta [line file line-number]) + +(defrecord Do [statements ret form env] + IAst + (children-keys [this] + `[:statements :ret])) + +(defrecord If [test then else form env] + IAst + (children-keys [this] + `[:test :then :else])) + +(defrecord Fn [name arities form env] + IAst + (children-keys [this] + `[:arities])) + +(defrecord Binding [type name form env] + IAst + (children-keys [this] + `[])) + +(defrecord FnBody [name arity args closed-overs variadic? body form env] + IAst + (children-keys [this] + `[:body])) + +(defrecord Let [bindings body form env] + IAst + (children-keys [this] + `[:bindings :body])) + +(defrecord LetBinding [name value form env] + IAst + (children-keys [this] + `[:value])) + +(defrecord Def [name value form env] + IAst + (children-keys [this] + `[:value])) + +(defrecord Recur [args env] + IAst + (children-keys [this] + `[:args])) + + +(defrecord LocalMacro [name respace form env]) + +(defrecord Invoke [args tail-call? form env] + IAst + (children-keys [this] + `[:args])) + +(defrecord Var [ns var-name form env] + IAst + (children-keys [this] + `[])) + +(defrecord VarConst [ns name form env] + IAst + (children-keys [this] + `[])) + +(defrecord Const [form env] + IAst + (children-keys [this] + `[])) + +(defrecord Vector [items form env] + IAst + (children-keys [this] + `[:items])) + +(defrecord Env [ns vars locals tail? meta bootstrap? recur-point]) + +;; Ctors + + +(defn make-invoke-ast [f args form env] + (->Invoke (cons f args) false form env)) + +(defn make-var-ast [ns name env] + (->Var ns name (symbol (pixie.stdlib/name name)) env)) + + +(defn make-var-const-ast [ns name env] + (->VarConst ns (symbol (pixie.stdlib/name name)) name env)) + +(defn make-invoke-var-ast [ns name args form env] + (make-invoke-ast + (make-var-ast ns name env) + args + form + env)) + +;; + +(defn convert-fn-body [name {:keys [variadic? args body form env] :as ast}] + (if variadic? + (make-invoke-var-ast + "pixie.stdlib" + (symbol "variadic-fn") + [(->Const (dec (count args)) env) + (convert-fn-body name (assoc ast :variadic? false))] + form + env) + ast)) + +(defprotocol ISimplfy + (simplify-ast [ast])) + +(extend-protocol ISimplfy + Do + (simplify-ast [{:keys [statements ret] :as ast}] + (if (= (count statements) 0) + ret + ast)) + + Let + (simplify-ast [{:keys [bindings body] :as ast}] + (if (= (count bindings) 0) + body + ast)) + + Fn + (simplify-ast [{:keys [name arities form env]}] + (if (= (count arities) 1) + (convert-fn-body name (first arities)) + (make-invoke-var-ast + "pixie.stdlib" + (symbol "multi-arity-fn") + (persistent! (reduce + (fn [acc {:keys [args variadic?] :as body}] + (-> acc + (conj! (->Const (if variadic? + -1 + (count args)) + env)) + (conj! (convert-fn-body name body)))) + (transient [(->Const (pixie.stdlib/name name) + env)]) + arities)) + form + env))) + + Invoke + (simplify-ast [{:keys [args env] :as ast}] + (let [first-arg (first args)] + (if (and (:tail? env) + (and (instance? Binding first-arg) + (= (:name first-arg) + (:recur-point env)))) + (->Recur args env) + ast))) + + Vector + (simplify-ast [{:keys [items form env]}] + (make-invoke-var-ast + "pixie.stdlib" + (if (:bootstrap? env) + (symbol "array") + (symbol "vector")) + items + form + env)) + + + Def + (simplify-ast [{:keys [name env value form]}] + (make-invoke-var-ast + "pixie.stdlib" + (symbol "set-var-root!") + [(make-var-const-ast @(:ns env) name env) + value] + form + env)) + + Object + (simplify-ast [this] + this)) diff --git a/pixie/bootstrap-macros.pxi b/pixie/bootstrap-macros.pxi new file mode 100644 index 00000000..7078db5e --- /dev/null +++ b/pixie/bootstrap-macros.pxi @@ -0,0 +1,219 @@ +(ns pixie.bootstrap-macros) + +;; +;; 1) The compiler should translate pixie.bootstrap-macros/foo to pixie.stdlib/foo +;; 2) the compiler should allow these macros to override any other macros/locals but only during bootstrap compilation +;; + +(defmacro when [test & body] + `(if ~test (do ~@body))) + + +(defmacro deftype + [nm fields & body] + (let [ctor-name (symbol (str "->" (name nm))) + fields (transduce (map (comp keyword name)) conj fields) + field-syms (transduce (map (comp symbol name)) conj fields) + mk-body (fn [body] + (let [fn-name (first body) + _ (assert (symbol? fn-name) "protocol override must have a name") + args (second body) + _ (assert (or (vector? args) + (seq? args)) "protocol override must have arguments") + self-arg (first args) + _ (assert (symbol? self-arg) "protocol override must have at least one `self' argument") + + rest (next (next body)) + body (reduce + (fn [body f] + `[(local-macro [~(symbol (name f)) + (get-field ~self-arg ~(keyword (name f)))] + ~@body)]) + rest + fields)] + `(fn ~(symbol (str fn-name "_" nm)) ~args ~@body))) + all-fields fields + type-nm (str #_@(:ns env) "." (name nm)) + type-decl `(def ~nm (create-type ~(keyword type-nm) + (array ~@all-fields))) + inst (gensym) + ctor `(defn ~ctor-name ~field-syms + (new ~nm + ~@field-syms)) + proto-bodies (transduce + (map (fn [body] + (cond + (symbol? body) `(satisfy ~body ~nm) + (seq? body) `(extend ~(first body) + ~(symbol (str #_@(:ns env) "/" nm)) + ~(mk-body body)) + :else (assert false "Unknown body element in deftype, expected symbol or seq")))) + conj + body)] + `(do ~type-decl + ~ctor + ~@proto-bodies))) + +(defmacro defprotocol + [nm & sigs] + `(do (def ~nm (~'pixie.stdlib/protocol ~(str nm))) + ~@(map (fn [[x]] + `(def ~x (~'pixie.stdlib/polymorphic-fn ~(str x) ~nm))) + sigs))) + + +(defmacro ns [nm & body] + (let [bmap (reduce (fn [m b] + (update-in m [(first b)] (fnil conj []) (next b))) + {} + body) + requires + (do + (assert (>= 1 (count (:require bmap))) + "Only one :require block can be defined per namespace") + (mapv (fn [r] `(require ~(keyword (name nm)) ~@r)) (first (:require bmap))))] + + `(do (in-ns ~(keyword (name nm))) + ~@requires))) + +(defmacro require [ins ns & args] + `(do (load-ns (quote ~ns)) + (assert (the-ns (quote ~ns)) + (str "Couldn't find the namespace " (quote ~ns) " after loading the file")) + + (apply refer ~ins (quote [~ns ~@args])))) + + +(defn -make-record-assoc-body [cname fields] + (let [k-sym (gensym "k") + v-sym (gensym "v") + this-sym (gensym "this") + result `(-assoc [~this-sym ~k-sym ~v-sym] + (condp identical? ~k-sym + ~@(mapcat + (fn [k] + [k `(~cname ~@(mapv (fn [x] + (if (= x k) + v-sym + `(get-field ~this-sym ~x))) + fields))]) + fields) + (throw [:pixie.stdlib/NotImplementedException + (str "Can't assoc to a unknown field: " ~k-sym)])))] + result)) + +(defmacro defrecord + {:doc "Define a record type. + +Similar to `deftype`, but supports construction from a map using `map->Type` +and implements IAssociative, ILookup and IObject." + :added "0.1"} + [nm field-syms & body] + (let [ctor-name (symbol (str "->" (name nm))) + map-ctor-name (symbol (str "map" (name ctor-name))) + fields (transduce (map (comp keyword name)) conj field-syms) + type-from-map `(defn ~map-ctor-name [m] + (apply ~ctor-name (map #(get m %) ~fields))) + default-bodies ['IAssociative + (-make-record-assoc-body ctor-name fields) + `(-contains-key [self# k#] + (contains? ~(set fields) k#)) + `(-dissoc [self k] + (throw [:pixie.stdlib/NotImplementedException + "dissoc is not supported on defrecords"])) + 'ILookup + `(-val-at [self# k# not-found#] + (get-field self# k# not-found#)) + 'IObject + `(-str [self# sb#] + (sb# (str "<" ~(name nm) " >" ))) + `(-eq [self other] + (and (instance? ~nm other) + ~@(map (fn [field] + `(= (. self ~field) (. other ~field))) + fields))) + `(-hash [self] + (hash [~@field-syms]))] + deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] + `(do ~type-from-map + ~deftype-decl))) + +(defmacro when [test & body] + `(if ~test (do ~@body))) + +(defmacro when-not [test & body] + `(if (not ~test) (do ~@body))) + +(defmacro when-let [binding & body] + (let [bind (nth binding 0 nil) + test (nth binding 1 nil)] + `(let [tmp# ~test] + (when tmp# + (let [~bind tmp#] + ~@body))))) + +(defmacro set! [v val] + `(-dynamic-var-set dynamic-var-handler (var ~v) ~val)) + +(defmacro binding [bindings & body] + (let [parted (partition 2 bindings)] + `(let [vars# (-dynamic-get-vars dynamic-var-handler)] + (with-handler [_# dynamic-var-handler] + (-dynamic-set-vars dynamic-var-handler + (assoc vars# + ~@(mapcat + (fn [[var-name binding]] + [`(var ~var-name) + binding]) + parted))) + ~@body)))) + + +(defmacro fn + {:doc "Creates a function. + +The following two forms are allowed: + (fn name? [param*] & body) + (fn name? ([param*] & body)+) + +The params can be destructuring bindings, see `(doc let)` for details."} + [& decls] + (let [name (if (symbol? (first decls)) [(first decls)] nil) + decls (if name (next decls) decls) + decls (cond + (vector? (first decls)) (list decls) + ;(satisfies? ISeqable (first decls)) decls + ;:else (throw (str "expected a vector or a seq, got a " (type decls))) + :else decls) + decls (seq (map (fn* [decl] + (let [argv (first decl) + names (vec (map #(if (= % '&) '& (gensym "arg__")) argv)) + bindings (loop [i 0 bindings []] + (if (< i (count argv)) + (if (= (nth argv i) '&) + (recur (inc i) bindings) + (recur (inc i) (reduce conj bindings [(nth argv i) (nth names i)]))) + bindings)) + body (next decl)] + (if (every? symbol? argv) + `(~argv ~@body) + `(~names + (let ~bindings + ~@body))))) + decls))] + (if (= (count decls) 1) + `(fn* ~@name ~(first (first decls)) ~@(next (first decls))) + `(fn* ~@name ~@decls)))) + + +(defmacro assert + ([test] + `(if ~test + nil + (throw [:pixie.stdlib/AssertionException + "Assert failed"]))) + ([test msg] + `(if ~test + nil + (throw [:pixie.stdlib/AssertionException + (str "Assert failed: " ~msg)])))) diff --git a/pixie/bootstrap.pxi b/pixie/bootstrap.pxi new file mode 100644 index 00000000..9552874c --- /dev/null +++ b/pixie/bootstrap.pxi @@ -0,0 +1,2957 @@ +;; This file is used to build what we need to even start running stdlib.pxi +;; Ordering of stuff will probably matter here + +(defprotocol ISeq + (-first [this]) + (-next [this])) + +(defprotocol ISeqable + (-seq [this])) + +(defprotocol ICounted + (-count [this])) + +(defprotocol IIndexed + (-nth [this idx]) + (-nth-not-found [this idx not-found])) + + +(defprotocol IPersistentCollection + (-conj [this x]) + (-disj [this x])) + +(defprotocol IEmpty + (-empty [this])) + +(defprotocol IObject + (-hash [this]) + (-eq [this other]) + (-str [this sbf]) + (-repr [this sbf])) + +(defprotocol IReduce + (-reduce [this f init])) + +(defprotocol IDeref + (-deref [this])) + +(defprotocol IReset + (-reset! [this val])) + +(defprotocol INamed + (-namespace [this]) + (-name [this])) + +(defprotocol IAssociative + (-assoc [this k v]) + (-contains-key [this k]) + (-dissoc [this k])) + +(defprotocol ILookup + (-val-at [this])) + +(defprotocol IMapEntry + (-key [this]) + (-val [this])) + +(defprotocol IStack + (-push [this v])) + +(defprotocol IPop + (-pop [this])) + +(defprotocol IFn + (-invoke [this args])) + +(defprotocol IDoc + (-doc [this])) + +(defprotocol IVector) +(defprotocol ISequential) +(defprotocol IMap) + +(defprotocol IMeta + (-with-meta [this x]) + (-meta [this])) + +(defprotocol ITransientCollection + (-conj! [this x])) + +(defprotocol IToTransient + (-transient [this x])) + +(defprotocol ITransient + (-persistent! [this])) + +(defprotocol ITransientStack + (-push! [this x]) + (-pop! [this])) + +(defprotocol IDisposable + (-dispose! [this])) + +(defprotocol IMessageObject + (-get-field [this name]) + (-invoke-method [this name args])) + + +(defprotocol IEffect + (-effect-val [this v]) + (-effect-finally [this v])) + +(defeffect EException + (-throw [this kw data ks])) + +(deftype ExceptionHandler [catches finally-fn] + IEffect + (-effect-val [this v] + v) + + (-effect-finally [this v] + (when finally-fn + (finally-fn)) + v) + + EException + (-throw [this kw data ks k] + (let [c (or (get catches kw) + (get catches :*))] + (println "Got Exception" kw c) + (if c + (c {:ex kw :data data :ks (conj ks k)}) + (-throw nil kw data (cons k ks)))))) + +(defn throw + ([[kw val]] + (throw kw val)) + ([kw val] + (-throw nil + kw + val + []))) + +(defn -try [body catches finally] + (with-handler [ex (->ExceptionHandler catches finally)] + (body))) + +(-run-external-extends) + +(extend -get-field Object -internal-get-field) +(extend -hash Object -internal-identity-hash) +(extend -meta Object (fn [x] nil)) + +(extend-type Object + (-str [x sb] + (sb (-internal-to-str x))) + + (-repr [x sb] + (sb (-internal-to-repr x))) + + (-eq [this other] + false)) + +(extend-type String + IObject + (-str [this sb] + (sb this))) + +;; Math wrappers + +(extend -eq Number -num-eq) + +(defn + + {:doc "Adds the arguments, returning 0 if no arguments" + :signatures [[& args]] + :added "0.1"} + ([] 0) + ([x] x) + ([x y] (-add x y)) + ([x y & more] + (-apply + (+ x y) more))) + +(defn - + ([] 0) + ([x] x) + ([x y] (-sub x y)) + ([x y & more] + (-apply - (- x y) more))) + +(defn * + ([] 1) + ([x] x) + ([x y] (-mul x y)) + ([x y & args] + (reduce -mul (-mul x y) args))) + +(defn / + ([x] (-div 1 x)) + ([x y] (-div x y)) + ([x y & args] + (reduce -div (-div x y) args))) + +(defn quot [num div] + (-quot num div)) + +(defn rem [num div] + (-rem num div)) + +(defn inc + ([x] (+ x 1))) + +(defn dec + ([x] (- x 1))) + +(defn < + ([x y] (-lt x y)) + ([x y & more] + (-apply < (< x y) more))) + +(defn > + ([x y] (-gt x y)) + ([x y & more] + (-apply > (> x y) more))) + + +(defn <= + ([x] true) + ([x y] (-lte x y)) + ([x y & rest] (if (-lte x y) + (apply <= y rest) + false))) + +(defn >= + ([x] true) + ([x y] (-gte x y)) + ([x y & rest] (if (-gte x y) + (apply >= y rest) + false))) + + +(defn = + {:doc "Returns true if all the arguments are equivalent. Otherwise, returns false. Uses +-eq to perform equality checks." + :signatures [[& args]] + :added "0.1"} + ([x] true) + ([x y] (if (identical? x y) + true + (-eq x y))) + ([x y & rest] (if (eq x y) + (apply = y rest) + false))) + +(defn pos? + {:doc "Returns true if x is greater than zero" + :signatures [[x]] + :added "0.1"} + [x] + (> x 0)) + +(defn neg? + {:doc "Returns true if x is less than zero" + :signatures [[x]] + :added "0.1"} + [x] + (< x 0)) + +(defn zero? + {:doc "Returns true if x is equal to zero" + :signatures [[x]] + :added "0.1"} + [x] + (= x 0)) + +(defn even? + {:doc "Returns true if n is even" + :signatures [[n]] + :added "0.1"} + [n] + (zero? (rem n 2))) + +(defn odd? + {:doc "Returns true of n is odd" + :signatures [[n]] + :added "0.1"} + [n] + (= (rem n 2) 1)) + + +;; Base functions + +(defn hash + [this] + (-hash this)) + +(defn identity + ^{:doc "The identity function. Returns its argument." + :added "0.1"} + ([x & _] x)) + +(defn not + [x] + (if x false true)) + +(defn not= + ([& args] + (not (-apply = args)))) + +(defn nil? + [x] + (identical? x nil)) + +(defn deref + [x] + (-deref x)) + +(defn conj + {:doc "Adds elements to the transient collection. Elements are added to the end except in the case of Cons lists" + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + ([] []) + ([coll] coll) + ([coll itm] (-conj coll itm)) + ([coll item & more] + (-apply conj (conj x y) more))) + + +(defn conj! + {:doc "Adds elements to the transient collection. Elements are added to the end except in the case of Cons lists" + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + ([] (-transient pixie.stdlib.persistent-vector/EMPTY)) + ([coll] (-persistent! coll)) + ([coll item] (-conj! coll item)) + ([coll item & args] + (reduce -conj! (-conj! coll item) args))) + +(defn disj + {:doc "Removes elements from the collection." + :signatures [[] [coll] [coll item]] + :added "0.1"} + ([] []) + ([coll] coll) + ([coll item] (-disj coll item)) + ([coll item & items] + (reduce -disj (-disj coll item) items))) + +(defn pop + {:doc "Pops elements off a stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + ([] []) + ([coll] (-pop coll))) + +(defn push + {:doc "Push an element on to a stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + ([coll x] (-push coll x))) + +(defn pop! + {:doc "Pops elements off a transient stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + ([coll] (-pop! coll))) + +(defn push! + {:doc "Push an element on to a transient stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + ([coll x] (-push! coll x))) + + +(defn nth + {:doc "Returns the element at the idx. If the index is not found it will return an error. + However, if you specify a not-found parameter, it will substitute that instead" + :signatures [[coll idx] [coll idx not-found]] + :added "0.1"} + ([coll idx] (-nth coll idx)) + ([coll idx not-found] (-nth-not-found coll idx not-found))) + +(defn get + {:doc "Get an element from a collection implementing ILookup, return nil or the default value if not found." + :added "0.1"} + ([mp k] + (get mp k nil)) + ([mp k not-found] + (-val-at mp k not-found))) + +(defn get-in + {:doc "Get a value from a nested collection at the \"path\" given by the keys." + :examples [["(get-in {:a [{:b 42}]} [:a 0 :b])" nil 42]] + :signatures [[m ks] [m ks not-found]] + :added "0.1"} + ([m ks] + (reduce get m ks)) + ([m ks not-found] + (loop [sentinel 'x + m m + ks (seq ks)] + (if ks + (let [m (get m (first ks) sentinel)] + (if (identical? sentinel m) + not-found + (recur sentinel m (next ks)))) + m)))) + +(defn contains? + [mp k] + (-contains-key mp k)) + +(defn name + [x] (-name x)) + +(defn namespace + [x] (-namespace x)) + + +(defn dispose! + "Finalizes use of the object by cleaning up resources used by the object" + [x] + (-dispose! x) + nil) + + +(defn has-meta? + [x] + (satisfies? IMeta x)) + +(defn meta + [x] + (-meta x)) + +(defn with-meta + [x m] + (-with-meta x m)) + +(defn count + ([coll] + (if (counted? coll) + (-count coll) + (loop [s (seq coll) + i 0] + (if (counted? s) + (+ i (count s)) + (recur (next s) + (inc i))))))) + +(defn assoc + {:doc "Associates the key with the value in the collection" + :signatures [[m] [m k v] [m k v & kvs]] + :added "0.1"} + ([m] m) + ([m k v] + (-assoc m k v)) + ([m k v & rest] + (apply assoc (-assoc m k v) rest))) + +(defn merge + ([a] a) + ([a b] + (reduce + (fn [a [k v]] + (assoc a k v)) + a + b)) + ([a b & cs] + (apply merge (merge a b) cs))) + +(defn assoc-in + {:doc "Associate a value in a nested collection given by the path. + +Creates new maps if the keys are not present." + :examples [["(assoc-in {} [:a :b :c] 42)" nil {:a {:b {:c 42}}}]] + :added "0.1"} + ([m ks v] + (let [ks (seq ks) + k (first ks) + ks (next ks)] + (if ks + (assoc m k (assoc-in (get m k) ks v)) + (assoc m k v))))) + +(defn update-in + {:doc "Update a value in a nested collection." + :examples [["(update-in {:a {:b {:c 41}}} [:a :b :c] inc)" nil {:a {:b {:c 42}}}]] + :added "0.1"} + [m ks f & args] + (let [f (fn [m] (apply f m args)) + update-inner-f (fn update-inner-f + ([m f k] + (assoc m k (f (get m k)))) + ([m f k & ks] + (assoc m k (apply update-inner-f (get m k) f ks))))] + (apply update-inner-f m f ks))) + +(defn key [m] + (-key m)) + +(defn val [m] + (-val m)) + +(defn keys + {:doc "If called with no arguments returns a transducer that will extract the key from each map entry. If passed + a collection, will assume that it is a hashmap and return a vector of all keys from the collection." + :signatures [[] [coll]] + :added "0.1"} + ([] (map key)) + ([m] + (with-handler [g (->Generator)] + (transduce (map key) yield g m)))) + +(defn vals + {:doc "If called with no arguments returns a transducer that will extract the key from each map entry. If passed + a collection, will assume that it is a hashmap and return a vector of all keys from the collection." + :signatures [[] [coll]] + :added "0.1"} + ([] (map val)) + ([m] + (with-handler [g (->Generator)] + (transduce (map val) yield g m)))) + + +(defn seq [x] + (-seq x)) + +(defn first [x] + (if (satisfies? ISeq x) + (-first x) + (let [x (seq x)] + (if (nil? x) + nil + (-first x))))) + +(defn second + {:doc "Returns the second item in coll, if coll implements IIndexed nth will be used to retrieve + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 1 nil) + (first (next coll)))) + +(defn third + {:doc "Returns the third item in coll, if coll implements IIndexed nth will be used to retrieve + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 2 nil) + (first (next (next coll))))) + +(defn fourth + {:doc "Returns the fourth item in coll, if coll implements IIndexed nth will be used to retrieve + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 3 nil) + (first (next (next (next coll)))))) + + +(defn next [x] + (if (satisfies? ISeq x) + (seq (-next x)) + (let [x (seq x)] + (if (nil? x) + nil + (seq (-next x)))))) + +(defn nthnext + {:doc "Returns the result of calling next n times on the collection." + :examples [["(nthnext [1 2 3 4 5] 2)" nil (3 4 5)] + ["(nthnext [1 2 3 4 5] 7)" nil nil]] + :added "0.1"} + [coll n] + (loop [n n + xs (seq coll)] + (if (and xs (pos? n)) + (recur (dec n) (next xs)) + xs))) + + +(defn apply [f & args] + (let [last-itm (last args) + but-last-cnt (dec (count args)) + arg-array (make-array (+ but-last-cnt + (count last-itm))) + idx (reduce + (fn [idx itm] + (aset arg-array idx itm) + (inc idx)) + but-last-cnt + last-itm)] + (array-copy args 0 arg-array 0 but-last-cnt) + (-apply f arg-array))) + +(defn fnil [f else] + (fn [x & args] + (apply f (if (nil? x) else x) args))) + +(defn comp + {:doc "Composes the given functions, applying the last function first." + :examples [["((comp inc first) [41 2 3])" nil 42]] + :signatures [[f] [f & fs]] + :added "0.1"} + ([] identity) + ([f] f) + ([f1 f2] + (fn [& args] + (f1 (apply f2 args)))) + ([f1 f2 f3] + (fn [& args] + (f1 (f2 (apply f3 args))))) + ([f1 f2 f3 & fs] + (fn [& args] + (apply (transduce comp (apply list f1 f2 f3 fs)) args)))) + +(defn last [coll] + (if (vector? coll) + (nth coll (dec (count coll))) + (loop [coll coll] + (if-let [v (next coll)] + (recur v) + (first coll))))) + +(defn butlast [coll] + (loop [res [] + coll coll] + (if (next coll) + (recur (conj res (first coll)) + (next coll)) + (seq res)))) + +(defn ith + {:doc "Returns the ith element of the collection, negative values count from the end. + If an index is out of bounds, will throw an Index out of Range exception. + However, if you specify a not-found parameter, it will substitute that instead" + :signatures [[coll i] [coll idx not-found]] + :added "0.1"} + ([coll i] + (when coll + (let [idx (if (neg? i) (+ i (count coll)) i)] + (nth coll idx)))) + ([coll i not-found] + (when coll + (let [idx (if (neg? i) (+ i (count coll)) i)] + (nth coll idx not-found))))) + +(defn take + {:doc "Takes n elements from the collection, or fewer, if not enough." + :added "0.1"} + [n coll] + (when (pos? n) + (when-let [s (seq coll)] + (cons (first s) (take (dec n) (next s)))))) + +(defn drop + {:doc "Drops n elements from the start of the collection." + :added "0.1"} + [n coll] + (let [s (seq coll)] + (if (and (pos? n) s) + (recur (dec n) (next s)) + s))) + +(defn repeat + ([x] + (cons x (lazy-seq (repeat x)))) + ([n x] + (take n (repeat x)))) + +(defn take-while + {:doc "Returns a lazy sequence of successive items from coll while + (pred item) returns true. pred must be free of side-effects. + Returns a transducer when no collection is provided." + :added "0.1"} + ([pred] + (fn [rf] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (if (pred input) + (rf result input) + (reduced result)))))) + ([pred coll] + (lazy-seq + (when-let [s (seq coll)] + (when (pred (first s)) + (cons (first s) (take-while pred (rest s)))))))) + + +(defn drop-while + {:doc "Returns a lazy sequence of the items in coll starting from the + first item for which (pred item) returns logical false. Returns a + stateful transducer when no collection is provided." + :added "0.1"} + ([pred] + (fn [rf] + (let [dv (atom true)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [drop? @dv] + (if drop? + (if (pred input) + result + (do + (reset! dv nil) + (rf result input))) + (rf result input)))))))) + ([pred coll] + (let [step (fn [pred coll] + (let [s (seq coll)] + (if (and s (pred (first s))) + (recur pred (rest s)) + s)))] + (lazy-seq (step pred coll))))) + +;; TODO: use a transient map in the future +(defn group-by + {:doc "Groups the collection into a map keyed by the result of applying f on each element. The value at each key is a vector of elements in order of appearance." + :examples [["(group-by even? [1 2 3 4 5])" nil {false [1 3 5] true [2 4]}] + ["(group-by (partial apply +) [[1 2 3] [2 4] [1 2]])" nil {6 [[1 2 3] [2 4]] 3 [[1 2]]}]] + :signatures [[f coll]] + :added "0.1"} + [f coll] + (reduce (fn [res elem] + (update-in res [(f elem)] (fnil conj []) elem)) + {} + coll)) + +;; TODO: use a transient map in the future +(defn frequencies + {:doc "Returns a map with distinct elements as keys and the number of occurences as values" + :added "0.1"} + [coll] + (reduce (fn [res elem] + (update-in res [elem] (fnil inc 0))) + {} + coll)) + +(defn partition + {:doc "Separates the collection into collections of size n, starting at the beginning, with an optional step size. + +The last element of the result contains the remaining element, not necessarily of size n if +not enough elements were present." + :examples [["(partition 2 [1 2 3 4 5 6])" nil ((1 2) (3 4) (5 6))] + ["(partition 2 [1 2 3 4 5])" nil ((1 2) (3 4) (5))] + ["(partition 2 1 [1 2 3 4 5])" nil ((1 2) (2 3) (3 4) (4 5) (5))]] + :signatures [[n coll] [n step coll]] + :added "0.1"} + ([n coll] (partition n n coll)) + ([n step coll] + (when-let [s (seq coll)] + (cons (take n s) (lazy-seq (partition n step (drop step s))))))) + +(defn partitionf + {:doc "A generalized version of partition. Instead of taking a constant number of elements, + this function calls f with the remaining collection to determine how many elements to + take." + :examples [["(partitionf first [2 :a, 3 :a :b, 4 :a :b :c])" + nil ((2 :a) (3 :a :b) (4 :a :b :c))]]} + [f coll] + (when-let [s (seq coll)] + (lazy-seq + (let [n (f s)] + (cons (take n s) + (partitionf f (drop n s))))))) + +(defn seq-reduce [s f acc] + (let [s (seq s)] + (if (reduced? acc) + @acc + (if (nil? s) + acc + (seq-reduce (next s) + f + (f acc (first s))))))) + +;; Some logic functions +(defn complement + {:doc "Given a function, return a new function which takes the same arguments + but returns the opposite truth value"} + [f] + (assert (fn? f) "Complement must be passed a function") + (fn + ([] (not (f))) + ([x] (not (f x))) + ([x y] (not (f x y))) + ([x y & more] (not (apply f x y more))))) + +;; + +;; Cons and List + +(deftype Cons [head tail meta] + IObject + (-str [this sb] + (sb "(") + (let [not-first (atom false)] + (reduce + (fn [_ x] + (if @not-first + (sb " ") + (reset! not-first true)) + (-str x sb)) + nil + this)) + (sb ")")) + + ISeq + (-first [this] head) + (-next [this] tail) + ISeqable + (-seq [this] this) + IMeta + (-meta [this] meta) + (-with-meta [this new-meta] + (->Cons head tail new-meta)) + + IReduce + (-reduce [this f init] + (seq-reduce this f init)) + + IIndexed + (-nth [self idx] + (loop [i idx + s (seq self)] + (if (or (= i 0) + (nil? s)) + (if (nil? s) + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"]) + (first s)) + (recur (dec i) + (next s))))) + + (-nth-not-found [self idx not-found] + (loop [i idx + s (seq self)] + (if (or (= i 0) + (nil? s)) + (if (nil? s) + not-found + (first s)) + (recur (dec i) + (next s)))))) + +(defn cons [head tail] + (assert (satisfies? ISeqable tail) (str "Can't seq " tail)) + (->Cons head tail nil)) + + +(deftype List [head tail cnt hash-val meta] + IObject + (-hash [this] + (if hash-val + hash-val + (let [val (reduce + pixie.stdlib.hashing/ordered-hashing-rf + this)] + (set-field! this :hash-val val) + val))) + + (-str [this sb] + (sb "(") + (let [not-first (atom false)] + (reduce + (fn [_ x] + (if @not-first + (sb " ") + (reset! not-first true)) + (-str x sb)) + nil + this)) + (sb ")")) + + + IIndexed + + (-nth [self idx] + (loop [i idx + s (seq self)] + (if (or (= i 0) + (nil? s)) + (if (nil? s) + (first s) + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"])) + (recur (dec i) + (next s))))) + + (-nth-not-found [self idx not-found] + (loop [i idx + s (seq self)] + (if (or (= i 0) + (nil? s)) + (if (nil? s) + not-found + (first s)) + (recur (dec i) + (next s))))) + + + IReduce + (-reduce [this f init] + (seq-reduce this f init)) + + + ISeq + (-first [this] head) + (-next [this] tail) + + ICounted + (-count [this] cnt) + + ISeqable + (-seq [this] this) + + IMeta + (-meta [this] meta) + (-with-meta [this new-meta] + (->List head tail cnt hash-val new-meta)) + + IPersistentCollection + (-conj [this val] + (->List val this (inc cnt) nil nil))) + + +(defn list [& args] + (loop [acc nil + idx (dec (count args)) + cnt 1] + (if (>= idx 0) + (recur (->List (nth args idx) + acc + cnt + nil + nil) + (dec idx) + (inc cnt)) + acc))) + +;; LazySeq start +(in-ns :pixie.stdlib.lazy-seq) + + +(deftype LazySeq [f s hash-val meta-data] + ISeqable + (-seq [this] + (sval this) + (when (.-s this) + (loop [ls (.-s this)] + (if (instance? LazySeq ls) + (recur (sval ls)) + (do (set-field! this :s ls) + (seq (.-s this))))))) + + ISeq + (-first [this] + (seq this) + (first (.-s this))) + + (-next [this] + (seq this) + (next (.-s this))) + + IReduce + (-reduce [this f init] + (seq-reduce this f init)) + + IMessageObject + (-get-field [this field] + (get-field this field))) + +(defn sval [this] + (if (.-f this) + (do (set-field! this :s ((.-f this))) + (set-field! this :f nil) + (.-s this)) + (.-s this))) + + +(in-ns :pixie.stdlib) + +(defn lazy-seq* [f] + (pixie.stdlib.lazy-seq/->LazySeq f nil nil nil)) + +(defeffect EGenerator + (-yield [this val])) + +(deftype Generator [] + IEffect + (-effect-val [this val] + nil) + (-effect-finally [this val] + val) + + EGenerator + (-yield [this val k] + (cons val (lazy-seq (k nil))))) + +(defn yield + ([g] nil) + ([g i] + (-yield g i) + g)) + +(defn sequence [coll] + (with-handler [gen (->Generator)] + (reduce yield gen coll))) + + + +;; LazySeq end + +;; String Builder + +(defn string-builder + ([] (-string-builder)) + ([sb] (-internal-to-str sb)) + ([sb x] + (if (instance? String x) + (-add-to-string-builder sb x) + (-str x (fn [x] + (-add-to-string-builder sb x)))))) + +(defn str + [& args] + (reduce + string-builder + args)) + + + +(defn println [& args] + (let [sb (-string-builder) + add-fn (fn [x] + (-add-to-string-builder sb x))] + (loop [idx 0 + sb sb] + (if (< idx (count args)) + (recur (inc idx) + (do (-str (aget args idx) add-fn) + (add-fn " ") + sb)) + (-blocking-println (-finish-string-builder sb)))) + nil)) + +;; + +;; Hashing Functions + +(in-ns :pixie.stdlib.hashing) + + +(def seed 0) +(def C1 (size-t 0xcc9e2d51)) +(def C2 (size-t 0x1b873593)) +(def LONG-BIT (size-t 32)) +(def MASK-32 (size-t 0xFFFFFFFF)) + +(defn mask-32 [x] + (bit-and x MASK-32)) + +(defn rotr [value shift] + (let [value (size-t value) + shift (size-t shift)] + (bit-or (bit-shift-left value shift) + (bit-shift-right value (- LONG-BIT shift))))) + +(defn rotl [value shift] + (let [value (size-t value) + shift (size-t shift)] + (bit-or (bit-shift-right value shift) + (bit-shift-left value (- LONG-BIT shift))))) + +(defn mix-k1 [k1] + (let [k1 (* k1 C1) + k1 (rotl k1 15) + k1 (* k1 C2)] + k1)) + +(defn mix-h1 [h1 k1] + (let [h1 (bit-xor h1 k1) + h1 (rotr h1 13) + h1 (+ (* h1 5) 0xe6546b64)] + h1)) + + +(defn fmix [h1 length] + (let [h1 (bit-xor h1 length) + h1 (bit-xor h1 (bit-shift-right h1 16)) + h1 (* h1 0x85ebca6b) + h1 (bit-xor h1 (bit-shift-right h1 13)) + h1 (* h1 0xc2b2ae35) + h1 (bit-xor h1 (bit-shift-right h1 16))] + h1)) + + +(defn mix-coll-hash [hash count] + (let [h1 seed + k1 (mix-k1 hash) + h1 (mix-h1 h1 k1)] + (fmix h1 count))) + +(deftype HashingState [n hash] + IMessageObject + (-get-field [this field] + (get-field this field))) + +(defn unordered-hashing-rf + ([] (->HashingState (size-t 0) (size-t 1))) + ([acc] + (mix-coll-hash (.-hash acc) + (.-n acc))) + ([acc itm] + (set-field! acc :n (inc (.-n acc))) + (set-field! acc :hash (+ (.-hash acc) + (hash itm))))) + +(defn ordered-hashing-rf + ([] (->HashingState 0 1)) + ([acc] + (mix-coll-hash (.-hash acc) + (.-n acc))) + ([acc itm] + (set-field! acc :n (inc (.-n acc))) + (set-field! acc :hash (+ (* (size-t 31) (.-hash acc)) + (hash itm))))) + + + +(defn hash-int [input] + (if (= input 0) + 0 + (let [k1 (mix-k1 input) + h1 (mix-h1 seed k1)] + (fmix h1 4)))) + +(defn hash-unencoded-chars [u] + (let [h1 (loop [i 1 + h1 seed] + (if (< i (count u)) + (let [k1 (bit-or (int (nth u (dec i))) + (bit-shift-left (int (nth u i)) 16)) + k1 (mix-k1 k1) + h1 (mix-h1 h1 k1)] + (recur (+ 2 i) + h1)) + h1)) + h1 (if (= (bit-and (count u) 1) 1) + (let [k1 (int (nth u (dec (count u)))) + k1 (mix-k1 k1) + h1 (bit-xor h1 k1)] + h1) + h1)] + (fmix h1 (* 2 (count u))))) + +(in-ns :pixie.stdlib) + +;; End Hashing Functions + + +;; Reduced + +(deftype Reduced [x] + IDeref + (-deref [this] x)) + +(defn reduced [x] + (->Reduced x)) + +(defn reduced? [x] + (instance? Reduced x)) + +;; End Reduced + +;; Basic Transducer Support + +(defn transduce + ([f coll] + (let [result (-reduce coll f (f))] + (f result))) + ([xform rf coll] + (let [f (xform rf) + result (-reduce coll f (f))] + (f result))) + ([xform rf init coll] + (let [f (xform rf) + result (-reduce coll f init)] + (f result)))) + +(defn reduce + ([rf col] + (rf (reduce rf (rf) col))) + ([rf init col] + (-reduce col rf init))) + +(defn into + ^{:doc "Add the elements of `from` to the collection `to`." + :signatures [[to from]] + :added "0.1"} + ([to from] + (if (satisfies? IToTransient to) + (persistent! (reduce conj! (transient to) from)) + (reduce conj to from))) + ([to xform from] + (if (satisfies? IToTransient to) + (transduce xform conj! (transient to) from) + (transduce xform conj to from)))) + +(defn vec + {:doc "Converts a reducable collection into a vector using the (optional) transducer." + :signatures [[coll] [xform coll]] + :added "0.1"} + ([coll] + (transduce conj coll)) + ([xform coll] + (transduce xform conj coll))) + +(defn map + ^{:doc "map - creates a transducer that applies f to every input element" + :signatures [[f] [f coll]] + :added "0.1"} + ([f] + (fn [xf] + (fn + ([] (xf)) + ([result] (xf result)) + ([result item] (xf result (f item)))))) + ([f coll] + (with-handler [g (->Generator)] + (transduce (map f) yield g coll))) + ([f & colls] + (let [step (fn step [cs] + (lazy-seq* + (fn [] + (let [ss (map seq cs)] + (if (every? identity ss) + (cons (map first ss) (step (map next ss))) + nil)))))] + (map (fn [args] (apply f args)) (step colls))))) + +(defn mapv + ([f col] + (transduce (map f) conj col))) + + +(defn filter + {:doc "Filter the collection for elements matching the predicate." + :signatures [[pred] [pred coll]] + :added "0.1"} + ([pred] + (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (if (pred i) + (xf acc i) + acc))))) + ([pred coll] + (with-handler [g (->Generator)] + (transduce (filter pred) yield g coll)))) + +(defn interpose + ^{:doc "Returns a transducer that inserts `val` in between elements of a collection." + :signatures [[val] [val coll]] + :added "0.1"} + ([val] (fn [xf] + (let [first? (atom true)] + (fn + ([] (xf)) + ([result] (xf result)) + ([result item] (if @first? + (do (reset! first? false) + (xf result item)) + (xf (xf result val) item))))))) + ([val coll] + (transduce (interpose val) conj coll))) + + +(def preserving-reduced + (fn preserving-reduced [rf] + (fn pr-inner [a b] + (let [ret (rf a b)] + (if (reduced? ret) + (reduced ret) + ret))))) + +(defn cat + {:doc "A transducer that concatenates elements of a collection." + :added "0.1"} + [rf] + (let [rrf (preserving-reduced rf)] + (fn cat-inner + ([] (rf)) + ([result] (rf result)) + ([result input] + (reduce rrf result input))))) + +(defn concat + {:doc "Concatenates its arguments." + :signatures [[& args]] + :added "0.1"} + [& args] (transduce cat conj args)) + +(defn mapcat + {:doc "Maps f over the elements of coll and concatenates the result" + :added "0.1"} + ([f] + (comp (map f) cat)) + ([f coll] + (transduce (mapcat f) conj coll))) + +(defn every? + {:doc "Check if every element of the collection satisfies the predicate." + :added "0.1"} + [pred coll] + (reduce + (fn [_ i] + (if (pred i) + true + (reduced false))) + true + coll)) + + +;; End Basic Transudcer Support + +;; Type Checks + +(defn instance? + {:doc "Checks if x is an instance of t. + + When t is seqable, checks if x is an instance of + any of the types contained therein." + :signatures [[t x]]} + [t x] + (if (-satisfies? ISeqable t) + (let [ts (seq t)] + (if (not ts) false + (if (-instance? (first ts) x) + true + (instance? (rest ts) x)))) + (-instance? t x))) + +(defn satisfies? + ^{:doc "Checks if x satisfies the protocol p. + + When p is seqable, checks if x satisfies all of + the protocols contained therein." + :signatures [[t x]]} + [p x] + (if (-satisfies? ISeqable p) + (let [ps (seq p)] + (if (not ps) true + (if (not (-satisfies? (first ps) x)) + false + (satisfies? (rest ps) x)))) + (-satisfies? p x))) + + +(defn true? [v] (identical? v true)) +(defn false? [v] (identical? v false)) + +(defn number? [v] (instance? Number v)) +(defn integer? [v] (instance? Integer v)) +(defn float? [v] (instance? Float v)) +(defn ratio? [v] (instance? Ratio v)) + +(defn char? [v] (instance? Character v)) +(defn string? [v] (instance? String v)) +(defn symbol? [v] (instance? Symbol v)) +(defn keyword? [v] (instance? Keyword v)) + +(defn list? [v] (instance? [PersistentList Cons] v)) +(defn seq? [v] (satisfies? ISeq v)) +(defn set? [v] (instance? pixie.stdlib.persistent-hash-set/PersistentHashSet v)) +(defn map? [v] (satisfies? IMap v)) +(defn fn? [v] (satisfies? IFn v)) + +(defn indexed? [v] (satisfies? IIndexed v)) +(defn counted? [v] (satisfies? ICounted v)) +(defn vector? [v] (satisfies? IVector v)) + +(defn int + {:doc "Converts a number to an integer." + :since "0.1"} + [x] + (cond + (integer? x) x + (float? x) (lround (floor x)) + (ratio? x) (int (/ (float (numerator x)) (float (denominator x)))) + (char? x) (-internal-int x) + :else (throw + [:pixie.stdlib/ConversionException + (throw (str "Can't convert a value of type " (type x) " to an Integer"))]))) + + +;; End Type Checks + +;; Range + +(in-ns :pixie.stdlib.range) + +(deftype Range [start stop step] + IReduce + (-reduce [self f init] + (loop [i start + acc init] + (if (or (and (> step 0) (< i stop)) + (and (< step 0) (> i stop)) + (and (= step 0))) + (let [acc (f acc i)] + (if (reduced? acc) + @acc + (recur (+ i step) acc))) + acc))) + ICounted + (-count [self] + (if (or (and (< start stop) (< step 0)) + (and (> start stop) (> step 0)) + (= step 0)) + 0 + (abs (quot (- start stop) step)))) + IIndexed + (-nth [self idx] + (when (or (= start stop 0) (neg? idx)) + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"])) + (let [cmp (if (< start stop) < >) + val (+ start (* idx step))] + (if (cmp val stop) + val + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"])))) + (-nth-not-found [self idx not-found] + (let [cmp (if (< start stop) < >) + val (+ start (* idx step))] + (if (cmp val stop) + val + not-found))) + ISeqable + (-seq [self] + (when (or (and (> step 0) (< start stop)) + (and (< step 0) (> start stop))) + (cons start (lazy-seq* #(range (+ start step) stop step))))) + IObject + (-str [this sbf] + (-str (seq this) sbf)) + (-repr [this sbf] + (-repr (seq this) sbf)) + (-eq [this sb] + nil)) + +(def MAX-NUMBER 0xFFFFFFFF) ;; 32 bits ought to be enough for anyone ;-) + +(in-ns :pixie.stdlib) + +(defn range + {:doc "Returns a range of numbers." + :examples [["(seq (range 3))" nil (0 1 2)] + ["(seq (range 3 5))" nil (3 4)] + ["(seq (range 0 10 2))" nil (0 2 4 6 8)] + ["(seq (range 5 -1 -1))" nil (5 4 3 2 1 0)]] + :signatures [[] [stop] [start stop] [start stop step]] + :added "0.1"} + ([] (pixie.stdlib.range/->Range 0 MAX-NUMBER 1)) + ([stop] (pixie.stdlib.range/->Range 0 stop 1)) + ([start stop] (pixie.stdlib.range/->Range start stop 1)) + ([start stop step] (pixie.stdlib.range/->Range start stop step))) + +;; End Range + +(in-ns :pixie.stdlib) + +;; Extend String + +(extend-type String + IIndexed + (-nth [self idx] + (-str-nth self idx)) + + ICounted + (-count [self] + (-str-len self)) + + IReduce + (-reduce [self f init] + (loop [acc init + idx 0] + (if (< idx (count self)) + (if (reduced? acc) + @acc + (recur (f acc (nth self idx)) + (inc idx))) + acc)))) + +;; End Extend String + +;; Extend Array + +(satisfy IVector Array) + +(extend-type Array + IObject + (-str [this sb] + (sb "[") + (let [not-first (atom false)] + (reduce + (fn [_ x] + (if @not-first + (sb " ") + (reset! not-first true)) + (-str x sb)) + nil + this)) + (sb "]")) + + IPersistentCollection + (-conj [arr itm] + (conj (pixie.stdlib.persistent-vector/vector-from-array arr) itm)) + + IIndexed + (-nth [this idx] + (if (and (<= 0 idx) + (< idx (count this))) + (aget this idx) + (throw [:pixie.stdlib/IndexOutOfRangeException + "Index out of range"]))) + (-nth-not-found [this idx not-found] + (if (and (<= 0 idx) + (< idx (count this))) + (aget this idx) + not-found)) + + ICounted + (-count ([arr] + (.-count arr))) + + IReduce + (-reduce [this f init] + (loop [idx 0 + acc init] + (if (reduced? acc) + @acc + (if (< idx (count this)) + (recur (inc idx) + (f acc (aget this idx))) + acc)))) + + ISeqable + (-seq [this] + (sequence this))) + +(extend-type ArrayMap + ILookup + (-val-at [this kw not-found] + (let [lst (array-map-to-array this)] + (loop [idx 0] + (if (< idx (count lst)) + (if (= (nth lst idx) kw) + (nth lst (inc idx)) + (recur (+ idx 2))) + not-found)))) + + IReduce + (-reduce [this f init] + (let [lst (array-map-to-array this)] + (loop [acc init + idx 0] + (if (reduced? acc) + @acc + (if (< idx (count lst)) + (let [k (nth lst idx) + v (nth lst (inc idx))] + (recur (f acc (map-entry k v)) + (+ idx 2))) + acc)))))) + +#_(defn array-copy [from from-idx to to-idx size] + (loop [idx 0] + (when (< idx size) + (do (aset to (+ to-idx idx) (aget from (+ from-idx idx))) + (recur (inc idx)))))) + +(defn array-append [arr val] + (let [new-array (make-array (inc (count arr)))] + (array-copy arr 0 new-array 0 (count arr)) + (aset new-array (count arr) val) + new-array)) + +(defn array-clone [arr] + (let [new-array (make-array (count arr))] + (array-copy arr 0 new-array 0 (count arr)) + new-array)) + +(defn array-resize [arr size] + (let [new-array (make-array size)] + (array-copy arr 0 new-array 0 size) + new-array)) + +(defn to-array [coll] + (let [arr (make-array (count coll))] + (reduce + (fn [idx i] + (aset arr idx i) + (inc idx)) + 0 + coll) + arr)) + +;;; Extend Var + +(extend-type Var + IObject + (-str [this sb] + (if (namespace this) + (do (sb "")) + (do (sb ""))))) + +;;; + +;; Atom + +(defprotocol IAtom + (-swap! [this f args]) + (-reset! [this val])) + +(deftype Atom [value] + IDeref + (-deref [this] + value) + + IAtom + (-swap! [this f args] + (let [new-val (apply f @this args)] + (-reset! this new-val) + new-val)) + (-reset! [this val] + (set-field! this :value val))) + +(defn atom [init] + (->Atom init)) + +(defn swap! [a f & args] + (-swap! a f args)) + +(defn reset! [a v] + (-reset! a v)) + +;; End Atom + +;; Map Entry + + + +(deftype MapEntry [k v meta] + IMapEntry + (-key [this] + k) + (-val [this] + v) + + IIndexed + (-nth [this idx] + (cond + (= idx 0) k + (= idx 1) v + :else (throw [:pixie.stdlib/IndexError + "Index out of Range"]))) + + (-nth-not-found [this idx not-found] + (cond + (= idx 0) k + (= idx 1) v + :else not-found))) + +(defn map-entry [k v] + (->MapEntry k v nil)) + +;; End Map Entry + + +;; PersistentVector + +(in-ns :pixie.stdlib.persistent-vector) + +(deftype Node [edit array] + IMessageObject + (-get-field [this name] + (get-field this name))) + +(defn new-node + ([edit] + (new-node edit (make-array 32))) + ([edit array] + (->Node edit array))) + +(def EMPTY-NODE (new-node nil)) + + + +(defn tail-off [this] + (let [cnt (.-cnt this)] + (if (< cnt 32) + 0 + (bit-shift-left (bit-shift-right (dec cnt) 5) 5)))) + +(defn array-for [this i] + (if (and (<= 0 i) (< i (.-cnt this))) + (if (>= i (tail-off this)) + (.-tail this) + (loop [node (.-root this) + level (.-shift this)] + (if (> level 0) + (recur (aget (.-array node) + (bit-and (bit-shift-right i level) 0x01f)) + (- level 5)) + (.-array node)))) + (throw [:pixie.stdlib/IndexOutOfRangeException + "Index out of range"]))) + +(deftype PersistentVector [cnt shift root tail hash-val meta] + IObject + (-hash [this] + (if hash-val + hash-val + (let [val (reduce + pixie.stdlib.hashing/ordered-hashing-rf + this)] + (set-field! this :hash-val val) + val))) + + (-str [this sb] + (sb "[") + (let [not-first (atom false)] + (reduce + (fn [_ x] + (if @not-first + (sb " ") + (reset! not-first true)) + (-str x sb)) + nil + this)) + (sb "]")) + + IVector + + ISeqable + (-seq [this] + (sequence this)) + + + IMessageObject + (-get-field [this name] + (get-field this name)) + + IPersistentCollection + (-conj [this val] + (assert (< cnt 0xFFFFFFFF) "Vector too large") + + (if (< (- cnt (tail-off this)) 32) + (let [new-tail (array-append tail val)] + (->PersistentVector (inc cnt) shift root new-tail hash-val meta)) + + (let [tail-node (->Node (.-edit root) tail)] + (if (> (bit-shift-right cnt 5) (bit-shift-left 1 shift)) + + (let [new-root (new-node (.-edit root)) + new-root-arr (.-array new-root)] + (aset new-root-arr 0 root) + (aset new-root-arr 1 (new-path (.-edit root) shift tail-node)) + (->PersistentVector (inc cnt) + (+ shift 5) + new-root + (array val) + hash-val + meta)) + (let [new-root (push-tail this shift root tail-node)] + (->PersistentVector (inc cnt) + shift + new-root + (array val) + hash-val + meta)))))) + IIndexed + (-nth [self i] + (if (and (<= 0 i) + (< i cnt)) + (let [node (array-for self i)] + (aget node (bit-and i 0x01F))) + (throw [:pixie.stdlib/IndexOutOfRange + (str "Index out of range, got " i " only have " cnt)]))) + + (-nth-not-found [self i not-found] + (if (and (<= 0 i) + (< i cnt)) + (let [node (array-for self i)] + (aget node (bit-and i 0x01F))) + not-found)) + + + + ILookup + (-val-at [this val] + (-nth-not-found self val nil)) + + + ICounted + (-count [this] cnt) + + IPop + (-pop [this] + (assert (not= cnt 0) "Can't pop an empty vector") + + (if (= cnt 1) + EMPTY + (if (> (- cnt (tail-off this)) 1) + (let [size (dec (count tail)) + new-tail (array-resize tail size)] + (->PersistentVector (dec cnt) + shift + root + new-tail + hash-val + meta)) + (let [new-tail (array-for this (- cnt 2)) + new-root (pop-tail this shift root)] + (cond + (nil? new-root) + (->PersisentVector (dec cnt) + shift + EMPTY-NODE + new-tail + hash-val + meta) + (and (> shift 5) + (nil? (aget (.-array new-root) 1))) + (->PersistentVector (dec cnt) + (- shift 5) + (aget (.-array new-root) 0) + new-tail + hash-val + meta) + + :else + (->PersistentVector (dec cnt) + shift + new-root + new-tail + hash-val + meta)))))) + + IAssociative + (-assoc [this k v] + (assert (int? k) "Vectors expect ints as keys") + (if (and (>= idx 0) + (< idx cnt)) + (if (>= idx (tail-off this)) + (let [new-tail (array-clone tail)] + (aset new-tail (bit-and idx 0x01f) val) + (->PersistentVector cnt shift root new-tail hash-val meta)) + (->PersistentVector cnt shift (do-assoc shift root idx val) tail hash-val meta)) + (if (= idx cnt) + (-conj this val) + (throw [:pixie.stdlib/IndexOutOfRange + "Can only assoc to the end or the contents of a vector"])))) + + IToTransient + (-transient [this] + (->TransientVector cnt shift + (editable-root root) + (editable-tail tail) + meta)) + + IReduce + (-reduce [this f init] + ((fn outer-loop [i acc] + (if (< i cnt) + (let [array (array-for this i)] + ((fn inner-loop [i j acc] + (if (< j (count array)) + (let [acc (f acc (aget array j))] + (if (reduced? acc) + @acc + (inner-loop (inc i) + (inc j) + acc))) + (outer-loop i acc))) + i 0 acc)) + acc)) + 0 init))) + +(defn do-assoc [lvl node idx val] + (let [new-array (array-clone (.-array node)) + ret (if (= lvl 0) + (aset new-array (bit-and idx 0x01f) val) + (let [sub-idx (bit-and (bit-shift-right idx lvl) 0x01f)] + (aset new-array sub-idx (do-assoc (- lvl 5) (aget (.-array node) idx val)))))] + (->Node (.-edit node) new-array))) + + +(defn push-tail [this level parent tail-node] + (let [subidx (bit-and (bit-shift-right (dec (.-cnt this)) level) 0x01f) + ret-array (array-clone (.-array parent)) + node-to-insert (if (= level 5) + tail-node + (let [child (aget (.-array parent) subidx)] + (if (= child nil) + (new-path (.-edit (.-root this)) + (- level 5) + tail-node) + (push-tail this + (- level 5) + child + tail-node))))] + (aset ret-array subidx node-to-insert) + (->Node (.-edit parent) ret-array))) + +(defn pop-tail [this level node] + (let [sub-idx (bit-and (bit-shift-right (dec (.-cnt this)) level) 0x01F)] + (cond + (> level 5) + (let [new-child (pop-tail this + (- level 5) + (aget (.-array node) sub-idx))] + (if (or (nil? new-child) + (= sub-idx 0)) + nil + (let [root (.-root this) + ret (->Node (.-edit root) + (.-array node))] + (aset (.-array ret) sub-idx new-child) + ret))) + + (= sub-idx 0) + nil + + :else + (let [root (.-root this) + ret (->Node (.-edit root) + (array-clone (.-array node)))] + (aset (.-array ret) sub-idx nil) + ret)))) + +(defn new-path [edit level node] + (if (= level 0) + node + (let [nnode (new-node edit)] + (aset (.-array nnode) 0 (new-path edit (- level 5) node)) + nnode))) + + +(def EMPTY (->PersistentVector 0 5 EMPTY-NODE (array 0) nil nil)) + +(defn vector-from-array [arr] + (if (< (count arr) 32) + (->PersistentVector (count arr) 5 EMPTY-NODE arr nil nil) + (into [] arr))) + + +;; Transient Vector + +#_(deftype Edited []) +(def edited ::edited) + + +(deftype TransientVector [cnt shift root tail meta] + IMessageObject + (-get-field [this field] + (get-field this field)) + + ITransientCollection + (-conj! [this val] + (ensure-editable this) + (let [i cnt] + (if (< (- i (tail-off this)) 32) + (do (aset tail (bit-and i 0x01F) val) + (set-field! this :cnt (inc cnt)) + this) + (let [tail-node (->Node (.-edit root) tail)] + (set-field! this :tail (make-array 32)) + (aset tail 0 val) + (if (> (bit-shift-right cnt 5) + (bit-shift-left 1 shift)) + (let [root-array (make-array 32)] + (aset root-array 0 root) + (aset root-array 1 (new-path (.-edit root) shift tail-node)) + (set-field! this :shift (+ shift 5)) + (set-field! this :root new-root) + (set-field! this :cnt (inc cnt)) + this) + (do (set-field this :root (push-tail-transient shift root tail-node)) + (set-field this :cnt (inc cnt)) + this)))))) + + ITransient + (-persistent! [this] + (ensure-editable this) + (let [trimmed (make-array (- cnt (tail-off self)))] + (array-copy tail 0 trimmed 0 (count trimmed)) + (->PersistentVector cnt shift root trimmed nil meta)))) + + +(defn editable-root [node] + (->Node edited (array-clone (.-array node)))) + +(defn ensure-editable [node] + (assert (nil? (.-edit node)) + "Transient used after call to persist!")) + +(defn ensure-node-editable [this node] + (let [root (.-root this)] + (if (identical? (.-edit node) (.-edit root)) + node + (->Node (.-edit root) (.-array node))))) + +(defn editable-tail [tail] + (let [ret (make-array 32)] + (array-copy tail 0 ret 0 (count tail)) + ret)) + +(in-ns :pixie.stdlib) +;; End Persistent Vector + +;; Persistent Hash Map + +(in-ns :pixie.stdlib.persistent-hash-map) + +(def MASK-32 0xFFFFFFFF) + + +(defprotocol INode + (-assoc-inode [this shift hash-val key val added-leaf]) + (-find [self shift hash-val key not-found]) + (-reduce-inode [self f init]) + (-without [self shift hash key])) + +(defn mask [hash shift] + (bit-and (bit-shift-right hash shift) 0x01f)) + +(defn bitpos [hash shift] + (bit-and (bit-shift-left 1 (mask hash shift)) MASK-32)) + +(defn index [this bit] + (bit-count32 (bit-and (.-bitmap this) + (dec bit)))) + + +(deftype BitmapIndexedNode [edit bitmap array] + IMessageObject + (-get-field [this field] + (get-field this field)) + + INode + (-assoc-inode [this shift hash-val key val added-leaf] + (let [bit (bitpos hash-val shift) + idx (index this bit)] + (if (not= (bit-and bitmap bit) 0) + (let [key-or-null (aget array (* 2 idx)) + val-or-node (aget array (inc (* 2 idx)))] + (if (nil? key-or-null) + (let [n (-assoc-inode val-or-node + (+ shift 5) + (bit-and hash-val MASK-32) + key + val + added-leaf)] + (if (identical? n val-or-node) + this + (->BitmapIndexedNode nil + bitmap + (clone-and-set array (inc (* 2 idx)) n)))) + (if (= key key-or-null) + (if (identical? val val-or-node) + this + (->BitmapIndexedNode nil + bitmap + (clone-and-set array (inc (* 2 idx)) val))) + (do (reset! added-leaf true) + (->BitmapIndexedNode nil bitmap + (clone-and-set2 array + (* 2 idx) nil + (inc (* 2 idx)) + (create-node (+ shift 5) + key-or-null + val-or-node + hash-val + key + val))))))) + (let [n (bit-count32 bitmap)] + (if (>= n 16) + (let [nodes (make-array 32) + jdx (mask hash-val shift)] + (aset nodes jdx (-assoc-inode BitmapIndexedNode-EMPTY + (+ shift 5) + hash-val + key + val + added-leaf)) + (loop [j 0 + i 0] + (when (< i 32) + (if (not= (bit-and (bit-shift-right bitmap i) 1) 0) + (do (if (nil? (aget array j)) + (aset nodes i (aget array (inc j))) + (aset nodes i (-assoc-inode BitmapIndexedNode-EMPTY + (+ shift 5) + (pixie.stdlib/hash (aget array j)) + (aget array j) + (aget array (inc j)) + added-leaf))) + (recur (+ 2 j) + (inc i))) + (recur j + (inc i))))) + (->ArrayNode nil (inc n) nodes)) + (let [new-array (make-array (* 2 (inc n)))] + (array-copy array 0 new-array 0 (* 2 idx)) + (aset new-array (* 2 idx) key) + (reset! added-leaf true) + (aset new-array (inc (* 2 idx)) val) + (array-copy array (* 2 idx) new-array (* 2 (inc idx)) (* 2 (- n idx))) + (->BitmapIndexedNode nil (bit-or bitmap bit) new-array))))))) + + (-find [self shift hash-val key not-found] + (let [bit (bitpos hash-val shift)] + (if (= (bit-and bitmap bit) 0) + not-found + (let [idx (index self bit) + key-or-null (aget array (* 2 idx)) + val-or-node (aget array (inc (* 2 idx)))] + (if (nil? key-or-null) + (-find val-or-node (+ 5 shift) hash-val key not-found) + (if (= key key-or-null) + val-or-node + not-found)))))) + + (-reduce-inode [this f init] + (loop [x 0 + acc init] + (if (< x (count array)) + (let [key-or-nil (aget array x) + val-or-node (aget array (inc x)) + acc (if (and (nil? key-or-nil) + (not (nil? val-or-node))) + (-reduce-inode val-or-node f acc) + (f acc (map-entry key-or-nil val-or-node)))] + (if (reduced? acc) + acc + (recur (+ 2 x) acc))) + acc)))) + +(def BitmapIndexedNode-EMPTY (->BitmapIndexedNode nil (size-t 0) [])) + +(defn ann [x] + (assert x) + x) + +(deftype ArrayNode [edit cnt array] + INode + (-assoc-inode [this shift hash-val key val added-leaf] + (let [idx (mask hash-val shift) + node (aget array idx)] + (if (nil? node) + (->ArrayNode nil + (inc cnt) + (clone-and-set array + idx + (-assoc-inode BitmapIndexedNode-EMPTY + (+ shift 5) + hash-val + key + val + added-leaf))) + (let [n (-assoc-inode node + (+ 5 shift) + hash-val + key + val + added-leaf)] + (if (identical? n node) + this + (->ArrayNode nil cnt (clone-and-set array idx n))))))) + (-find [this shift hash-val key not-found] + (let [idx (mask hash-val shift) + node (aget array idx)] + (if (nil? node) + not-found + (-find node + (+ shift 5) + hash-val + key + not-found)))) + + (-reduce-inode [this f init] + (loop [x 0 + acc init] + (if (< x (count array)) + (let [node (aget array x)] + (if (not (nil? node)) + (let [acc (-reduce-inode node f acc)] + (if (reduced? acc) + acc + (recur (inc x) acc))) + (recur (inc x) acc))) + acc)))) + +(deftype PersistentHashMap [cnt root has-nil? nil-val hash-val meta] + IMap + IFn + (-invoke [this k] + (-val-at this k nil)) + + IObject + (-hash [this] + (if hash-val + hash-val + (do (set-field! this :hash-val + (transduce cat + pixie.stdlib.hashing/unordered-hash-reducing-fn + this)) + hash-val))) + + (-str [this sb] + (sb "{") + (let [not-first (atom false)] + (reduce + (fn [_ x] + (if @not-first + (sb " ") + (reset! not-first true)) + (-str (key x) sb) + (sb " ") + (-str (val x) sb)) + nil + this)) + (sb "}")) + + IMeta + (-meta [this] + meta) + (-with-meta [this new-meta] + (->PersistentHashMap cnt root has-nil? nil-val hash-val new-meta)) + + IEmpty + (-empty [this] + (-with-meta pixie.stdlib.persistent-hash-map/EMPTY + meta)) + + IAssociative + (-assoc [this key val] + (if (nil? key) + (if (identical? val nil-val) + this + (->PersistentHashMap cnt root true val nil meta)) + (let [new-root (if (nil? root) + BitmapIndexedNode-EMPTY + root) + added-leaf (atom false) + new-root (-assoc-inode new-root + 0 + (bit-and (hash key) MASK-32) + key + val + added-leaf)] + (if (identical? new-root root) + this + (->PersistentHashMap (if @added-leaf + (inc cnt) + cnt) + new-root + has-nil? + nil-val + nil + meta))))) + + (-contains-key [this k] + (if (identical? (-val-at this k NOT-FOUND) NOT-FOUND) + false + true)) + + ILookup + (-val-at [this key not-found] + (if (nil? key) + (if has-nil? + nil-val + not-found) + (if (nil? root) + not-found + (-find root 0 + (bit-and (hash key) MASK-32) + key + not-found)))) + + ISeqable + (-seq [this] + (sequence this)) + + IReduce + (-reduce [this f init] + (let [acc (if has-nil? + (f init nil-val) + init)] + (if (reduced? acc) + @acc + (if root + (let [acc (-reduce-inode root f acc)] + (if (reduced? acc) + @acc + acc)) + acc))))) + +(deftype NOT-FOUND-TP []) +(def NOT-FOUND (->NOT-FOUND-TP)) + +(def EMPTY (->PersistentHashMap (size-t 0) nil false nil nil nil)) + +(defn create-node [shift key1 val1 key2hash key2 val2] + (let [key1hash (bit-and (pixie.stdlib/hash key1) MASK-32)] + (if (= key1hash key2hash) + (do + (println "HASH " key1 val1 key2 val2 key1hash key2hash) + (->HashCollisionNode nil key1hash [key1 val1 key2 val2])) + (let [added-leaf (atom false)] + (-> BitmapIndexedNode-EMPTY + (-assoc-inode shift key1hash key1 val1 added-leaf) + (-assoc-inode shift key2hash key2 val2 added-leaf)))))) + +(defn clone-and-set [array i a] + (let [clone (array-clone array)] + (aset clone i a) + clone)) + +(defn clone-and-set2 [array i a j b] + (let [clone (array-clone array)] + (aset clone i a) + (aset clone j b) + clone)) + +(in-ns :pixie.stdlib) + +(defn hashmap [& args] + (loop [idx 0 + acc pixie.stdlib.persistent-hash-map/EMPTY] + (if (< idx (count args)) + (do (assert (> (- (count args) idx) 1) "hashmap requires even number of args") + (recur (+ 2 idx) + (assoc acc (nth args idx) (nth args (inc idx))))) + acc))) + +;; End Persistent Hash Map + +;; Start Persistent Hash Set + +(in-ns :pixie.stdlib.persistent-hash-set) + +(deftype PersistentHashSet [m hash-val meta] + IObject + (-hash [this] + (when-not hash-val + (set-field! this :hash-val (reduce + unordered-hashing-rf + this))) + hash-val) + + (-str [this sb] + (sb "#{") + (let [not-first (atom false)] + (reduce + (fn [_ x] + (if @not-first + (sb " ") + (reset! not-first true)) + (-str x sb)) + nil + this)) + (sb "}")) + + ICounted + (-count [this] + (-count m)) + + IPersistentCollection + (-conj [this x] + (->PersistentHashSet (-assoc m x x) nil meta)) + (-disj [this x] + (->PersistentHashSet (-dissoc m x) nil meta)) + + IMeta + (-meta [this] + meta) + + (-with-meta [this x] + (->PersistentHashSet m hash-val x)) + + IAssociative + (-contains-key [this x] + (-contains-key m x)) + + ILookup + (-val-at [this k not-found] + (-val-at m k not-found)) + + IReduce + (-reduce [this f init] + (reduce + (fn [acc kv] + (f acc (key kv))) + init + m))) + +(def EMPTY (->PersistentHashSet + pixie.stdlib.persistent-hash-map/EMPTY + nil + nil)) + +;; End Persistent Hash Set + +(in-ns :pixie.stdlib) + +(defn set [coll] + (into pixie.stdlib.persistent-hash-set/EMPTY + coll)) + + +;; Extend Core Types + +(extend -invoke Code -invoke) +(extend -invoke NativeFn -invoke) +(extend -invoke VariadicCode -invoke) +(extend -invoke MultiArityFn -invoke) +(extend -invoke Closure -invoke) +(extend -invoke Var -invoke) +(extend -invoke PolymorphicFn -invoke) +(extend -invoke DoublePolymorphicFn -invoke) + +(extend -name Keyword -internal-get-name) +(extend -namespace Keyword -internal-get-ns) +(extend -hash Keyword (fn [x] + (let [v (-internal-get-hash x)] + (if (zero? v) + (let [h (pixie.stdlib.hashing/hash-unencoded-chars (str x))] + (-internal-store-hash x h) + h) + v)))) + +(extend -invoke Keyword (fn + ([this o] + (-val-at o this nil)) + ([this o not-found] + (-val-at o this not-found)))) + + + +(extend -name Symbol -internal-get-name) +(extend -namespace Symbol -internal-get-ns) +(extend -hash Symbol (fn [x] + (let [v (-internal-get-hash x)] + (if (zero? v) + (let [h (pixie.stdlib.hashing/hash-unencoded-chars (str x))] + (-internal-store-hash x h) + h) + v)))) + +(extend -name String identity) +(extend -namespace String (fn [x] nil)) + + + + + +;(extend -reduce Cons seq-reduce) +;(extend -reduce PersistentList seq-reduce) +;(extend -reduce LazySeq seq-reduce) + + + +;; Buffer + +(extend -reduce Buffer + (fn [b f init] + (loop [idx 0 + acc init] + (if (reduced? acc) + @acc + (if (< idx (count b)) + (let [val (pixie.ffi/unpack b idx CUInt8)] + (recur (inc idx) + (f acc val))) + acc))))) + +;; + + +(extend -str Bool + (fn [x sb] + (if x + (sb "true") + (sb "false")))) +(extend -repr Bool -str) + +(extend-type Nil + IObject + (-str [this f] + (f "nil")) + (-repr [this f] + (f "nil")) + (-hash [this] + 1000000) + (-eq [this other] + (identical? this other)) + + IPersistentCollection + (-conj [this x] + (vector x)) + (-disj [this x] + nil) + + IAssociative + (-assoc [this k v] + (hashmap k v)) + (-dissoc [this k] + nil) + (-contains-key [this k] + false) + + ILookup + (-val-at + ([this k] nil) + ([this k not-found] nil)) + + IDeref + (-deref [this] + nil) + + IIndexed + + (-nth [this idx] + (throw [:pixie.stdlib/IndexOutOfRangeException + "Index out of range"])) + (-nth-not-found [this idx not-found] + not-found) + + IReduce + (-reduce [this f init] + init) + + ICounted + (-count [this] + 0) + + ISeqable + (-seq [this] + nil) + + ISeq + (-first [this] + nil) + (-next [this] + nil)) + + +(extend -with-meta Nil (fn [self _] nil)) +(extend -contains-key Nil (fn [_ _] false)) + +(extend -hash Integer pixie.stdlib.hashing/hash-int) + +(extend -eq Integer -num-eq) +(extend -eq Float -num-eq) +(extend -eq Ratio -num-eq) + +(defn vector [& args] + (pixie.stdlib.persistent-vector/vector-from-array args)) + + +;; End Extend Core Types + +;; NS functions + +(defn refer + {:doc "Refer to the specified vars from a namespace directly. + +Supported filters: + :rename refer to the given vars under a different name + :exclude don't refer the given vars + :refer + :all refer all vars + :refer refer only the given vars + :only same as refer + +user => (refer 'pixie.string :refer :all) +user => (refer 'pixie.string :only '(index-of starts-with? ends-with?)) +user => (refer 'pixie.string :rename '{index-of find}) +user => (refer 'pixie.string :exclude '(substring))" + :added "0.1"} + [from-ns ns-sym & filters] + (assert ns-sym "Must provide a ns-sym") + (assert from-ns "Must provide a from-ns") + (let [ns (or (the-ns ns-sym) (throw [:pixie.stdlib/NamespaceNotFoundException + (str "No such namespace: " ns-sym)])) + filters (apply hashmap filters) + rename (or (:rename filters) {}) + exclude (:exclude filters) + rname (or (:as filters) + ns-sym)] + (-add-refer from-ns ns-sym rname) + (if (= :all (:refer filters)) + (-refer-all from-ns rname) + (doseq [sym (:refer filters)] + (-add-refer from-ns rname sym sym))) + + (doseq [[from to] (:rename filters)] + (-add-refer from-ns rname from to)) + + (doseq [nm (:exclude filters)] + (-add-exclude from-ns rname nm)) + + nil)) + + +(defn load-ns [ns-sym] + (assert (not (namespace ns-sym)) "Namespace names must not be namespaced") + (or (the-ns ns-sym) + (load-file ns-sym))) + +;; End NS Functions + +;; Delay + +(deftype Delay [f val] + (-deref [this] + (when f + (set-field! :val (f)) + (set-field! :f nil)) + val)) + +(defn -delay [f] + (->Delay f nil)) + +;; End Delay + +;; Dynamic Vars + +(defeffect EDynamicVarEnv + (-dynamic-var-get [this var]) + (-dynamic-var-set [this var val]) + (-dynamic-get-vars [this]) + (-dynamic-set-vars [this s'])) + +(deftype DynamicVar [x] + IEffect + (-effect-val [this y] + (fn val-fn [s] y)) + + (-effect-finally [this f] + (f x)) + + EDynamicVarEnv + (-dynamic-var-get [this var k] + (fn lookup-fn [s] + ((k (get s var)) s))) + + (-dynamic-var-set [this var val k] + (fn [s] + ((k nil) (assoc s var val)))) + + (-dynamic-get-vars [this k] + (fn [s] + ((k s) s))) + + (-dynamic-set-vars [this s' k] + (fn set-vars [s] + ((k nil) s')))) + + +(def dynamic-var-handler (->DynamicVar {})) + + + + + + +;; LibC Stuff +(def load-paths (atom ["." ""])) + +(defn ffi-library [nm] + (reduce + (fn [_ x] + (when-let [l (-ffi-library (str x (name nm)))] + (reduced l))) + nil + @load-paths)) + + +(def libc (ffi-library pixie.platform/lib-c-name)) + + +;; + + + +;; Mutimethods + +(defmacro defmulti + {:doc "Define a multimethod, which dispatches to its methods based on dispatch-fn." + :examples [["(defmulti greet first)"] + ["(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\"))"] + ["(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".\"))"] + ["(greet [:hi \"Jane\"])" nil "Hi, Jane!"]] + :signatures [[name dispatch-fn & options]] + :added "0.1"} + [name & args] + (let [[meta args] (if (string? (first args)) + [{:doc (first args)} (next args)] + [{} args]) + [meta args] (if (map? (first args)) + [(merge meta (first args)) (next args)] + [meta args]) + dispatch-fn (first args) + options (apply hashmap (next args))] + `(def ~name (->MultiMethod ~dispatch-fn ~(get options :default :default) (atom {}))))) + +(defmacro defmethod + {:doc "Define a method of a multimethod. See `(doc defmulti)` for details." + :signatures [[name dispatch-val [param*] & body]] + :added "0.1"} + [name dispatch-val params & body] + `(do + (let [methods (.-methods ~name)] + (swap! methods + assoc + ~dispatch-val (fn ~params + ~@body)) + ~name))) + +(deftype MultiMethod [dispatch-fn default-val methods] + IFn + (-invoke [self & args] + (let [dispatch-val (apply dispatch-fn args) + method (or (get @methods dispatch-val) + (get @methods default-val)) + _ (assert method (str "no method defined for " dispatch-val))] + (apply method args)))) + + +;; End Multimethods + +(def gensym-state (atom 0)) +(defn gensym + ([] (gensym "auto_")) + ([prefix] (symbol (str prefix (swap! gensym-state inc))))) + + +;; Macros + +(defmacro fn + {:doc "Creates a function. + +The following two forms are allowed: + (fn name? [param*] & body) + (fn name? ([param*] & body)+) + +The params can be destructuring bindings, see `(doc let)` for details."} + [& decls] + (let [name (if (symbol? (first decls)) [(first decls)] nil) + decls (if name (next decls) decls) + decls (cond + (vector? (first decls)) (list decls) + ;(satisfies? ISeqable (first decls)) decls + ;:else (throw (str "expected a vector or a seq, got a " (type decls))) + :else decls) + decls (seq (map (fn [[argv & body]] + (let [names (vec (map #(if (= % '&) '& (gensym "arg__")) argv)) + bindings (loop [i 0 bindings []] + (if (< i (count argv)) + (if (= (nth argv i) '&) + (recur (inc i) bindings) + (recur (inc i) (reduce conj bindings [(nth argv i) (nth names i)]))) + bindings))] + (if (every? symbol? argv) + `(~argv ~@body) + `(~names + (let ~bindings + ~@body))))) + decls))] + (if (= (count decls) 1) + `(fn* ~@name ~(first (first decls)) ~@(next (first decls))) + `(fn* ~@name ~@decls)))) + + +(defmacro defn ^{:doc "Defines a new function." + :signatures [[nm doc? meta? & body]]} + [nm & rest] + (let [meta (if (meta nm) (meta nm) {}) + meta (if (instance? String (first rest)) + (assoc meta :doc (first rest)) + meta) + rest (if (instance? String (first rest)) (next rest) rest) + meta (if (satisfies? IMap (first rest)) + (merge meta (first rest)) + meta) + rest (if (satisfies? IMap (first rest)) (next rest) rest) + meta (if (-contains-key meta :signatures) + meta + (merge meta {:signatures + (if (satisfies? IVector (first rest)) + [(first rest)] + (transduce (map first) conj rest))})) + nm (with-meta nm meta)] + `(def ~nm (fn ~nm ~@rest)))) + + +(defn destructure [binding expr] + (cond + (symbol? binding) [binding expr] + (vector? binding) (let [name (gensym "vec__")] + (reduce conj [name expr] + (destructure-vector binding name))) + (map? binding) (let [name (gensym "map__")] + (reduce conj [name expr] + (destructure-map binding name))) + :else (throw [:pixie.stdlib/AssertionException + (str "unsupported binding form: " binding)]))) + +(defn destructure-vector [binding-vector expr] + (loop [bindings (seq binding-vector) + i 0 + res []] + (if bindings + (let [binding (first bindings)] + (cond + (= binding '&) (recur (nnext bindings) + (inc (inc i)) + (reduce conj res (destructure (second bindings) `(nthnext ~expr ~i)))) + (= binding :as) (reduce conj res (destructure (second bindings) expr)) + :else (recur (next bindings) + (inc i) + (reduce conj res (destructure (first bindings) `(nth ~expr ~i nil)))))) + res))) + +(defn destructure-map [binding-map expr] + (let [defaults (or (:or binding-map) {}) + res + (loop [bindings (seq binding-map) + res []] + (if bindings + (let [binding (key (first bindings)) + binding-key (val (first bindings))] + (if (contains? #{:or :as :keys} binding) + (recur (next bindings) res) + (recur (next bindings) + (reduce conj res (destructure binding `(get ~expr ~binding-key ~(get defaults binding))))))) + res)) + expand-with (fn [convert] #(vector % `(get ~expr ~(convert %) ~(get defaults %)))) + res (if (contains? binding-map :keys) (transduce (map (expand-with (comp keyword name))) concat res (get binding-map :keys)) res) + res (if (contains? binding-map :as) + (reduce conj res [(get binding-map :as) expr]) + res)] + res)) + +(defmacro let + {:doc "Makes the bindings availlable in the body. + +The bindings must be a vector of binding-expr pairs. The binding can be a destructuring +binding, as below. + +Vector destructuring: + [x y z] binds the first three elements of the collection to x, y and z + [x y & rest] binds rest to the elements after the first two elements of the collection + [x y :as v] binds the value of the complete collection to v + +Map destructuring: + {a :a, b :b} binds a and b to the values associated with :a and :b + {a :a :as m} binds the value of the complete collection to m + {a :a :or {a 42}} binds a to the value associated with :a, or 42, if not present + {:keys [a b c]} binds a, b and c to the values associated with :a, :b and :c + +All these forms can be combined and nested, in the example below: + +(let [[x y [z :as iv] :as v] [1 2 [3 4 5] 6 7] + {a :a [b c {:keys [d]}] :more :or {a 42}} {:a 1, :more [1 2 {:d 3, :e 4}]}] + ...) + +For more information, see http://clojure.org/special_forms#binding-forms"} + [bindings & body] + (let* [destructured-bindings (transduce (map (fn let-foo [args] + (assert (= 2 (count args)) (str "Bindings must be in pairs, not " args + " " (meta (first args)))) + (apply destructure args))) + concat + [] + (partition 2 bindings))] + `(let* ~destructured-bindings + ~@body))) + +(defmacro defmacro + {:doc "Defines a new macro." + :added "0.1"} + [nm & rest] + `(do (defn ~nm ~@rest) + (set-macro! ~nm) + ~nm)) + +;; State + +(defeffect EState + (-state-lookup [this k]) + (-state-update [this key f args])) + +(deftype State [x] + IEffect + (-effect-val [this y] + (fn val-fn [s] y)) + + (-effect-finally [this f] + (f x)) + + EState + (-state-lookup [this key k] + (fn lookup-fn [s] + ((k (get s key)) s))) + + (-state-update [this key f args k] + (fn [s] + (let [new-state (assoc s key (apply f (get s key) args))] + ((k nil) new-state)))) + + + ILookup + (-val-at [this k] + (-sate-lookup this k))) + +(defn state + ([] + (state {})) + ([s] + (->State s))) + +(defn update [s k f & args] + (-state-update s k v)) + +(defmacro with-state [[n] & body] + `(with-handler [~n (state)] + ~@body)) + +;; String stuff +(in-ns :pixie.string) + +(defn substring + ([s start] + (substring s start (count s))) + ([s start end] + (-substring s start end))) + + +(in-ns :pixie.stdlib) +;; End String diff --git a/pixie/compile-bootstrap.pxi b/pixie/compile-bootstrap.pxi new file mode 100644 index 00000000..bd062415 --- /dev/null +++ b/pixie/compile-bootstrap.pxi @@ -0,0 +1,48 @@ +(ns pixie.compile-bootstrap + (:require [pixie.compiler :refer :all] + [pixie.io :as io] + [pixie.pxic-writer :as pxic-writer] + [pixie.bootstrap-macros])) + + + +(defn read-and-compile [form env] + (let [ast (analyze form env)] + ast)) + +(defn compile-file [env from os] + (let [forms (read-string (str "[" (io/slurp from) "]") from) + form-count (atom 0) + total-count (atom 0)] + (doseq [form forms] + (swap! form-count inc) + (swap! total-count inc) + (when (= @form-count 10) + (println from (int (* 100 (/ @total-count (count forms)))) "% in" @(:ns env)) + (reset! form-count 0)) + + (let [ast (read-and-compile form env)] + (pxic-writer/write-object os ast))))) + +(defn compile-files [files to] + (let [os (-> to + io/open-write + io/buffered-output-stream) + env (new-env true)] + + (binding [pxic-writer/*cache* (pxic-writer/writer-cache os)] + (doseq [file files] + (compile-file env file os))) + (dispose! os))) + +(compile-files ["pixie/bootstrap.pxi" + "pixie/bootstrap-macros.pxi" + "pixie/streams.pxi" + "pixie/io-blocking.pxi" + "pixie/reader.pxi" + "pixie/ast.pxi" + "pixie/ast-output.pxi" + "pixie/compiler.pxi" + "pixie/main.pxi"] + "./bootstrap.pxic") + diff --git a/pixie/compiler.pxi b/pixie/compiler.pxi new file mode 100644 index 00000000..1caaedab --- /dev/null +++ b/pixie/compiler.pxi @@ -0,0 +1,395 @@ +(ns pixie.compiler + (:require [pixie.string :as string] + [pixie.ast :refer :all])) + + +(defn remove-bootstrap [env sym] + (if (= (namespace x) "pixie.bootstrap-macros") + (symbol (str "pixie.stdlib/" (name x))) + x)) + +(defmulti analyze-form (fn [_ x] + (cond + (identical? x true) :bool + (identical? x false) :bool + (keyword? x) :keyword + (nil? x) nil + (seq? x) :seq + (vector? x) :vector + (symbol? x) :symbol + (number? x) :number + (string? x) :string + (char? x) :char + (map? x) :map + (set? x) :set + :else (do (println "Cant' Analyze " x) + (assert false "Can't analyze"))))) + +(defmulti analyze-seq (fn [_ x] + (let [f (first x)] + (if (symbol? f) + (let [sname (symbol (name f))] + (if (get @(get-field analyze-seq :methods) sname) + sname + f)) + :invoke)))) + + +;; Special Forms + +(defmethod analyze-seq 'do + [env x] + (let [statement-asts (mapv (partial analyze-form (assoc env :tail? false)) + (butlast (next x)))] + (->Do statement-asts + (analyze-form env (last x)) + x + env))) + +(defmethod analyze-seq 'comment + [env x] + (->Const nil env)) + +(defmethod analyze-seq 'if + [env [_ test then else :as form]] + (->If (analyze-form (assoc env :tail? false) test) + (analyze-form env then) + (analyze-form env else) + form + env)) + +(defmethod analyze-seq 'this-ns-name + [env [_]] + (analyze-form env (name @(:ns env)))) + +(defmethod analyze-seq 'with-handler + [env [_ [h handler] & body]] + (analyze-form env + `(let [~h ~handler] + (~'pixie.stdlib/-effect-finally + ~h + (~'pixie.stdlib/-with-handler + ~h + (fn [] + (~'pixie.stdlib/-effect-val + ~h + (do ~@body)))))))) + +(defmethod analyze-seq 'defeffect + [env [_ nm & sigs]] + (analyze-form env + `(do (def ~nm (~'pixie.stdlib/protocol ~(str nm))) + ~@(map (fn [[x]] + `(def ~x (~'pixie.stdlib/-effect-fn + (~'pixie.stdlib/polymorphic-fn ~(str x) ~nm)))) + sigs)))) + + +(defmethod analyze-seq 'fn* + [env [_ & body :as form]] + (let [[name body] (if (symbol? (first body)) + [(first body) + (next body)] + [(gensym "ann-fn_") + body]) + arities (if (vector? (first body)) + [body] + body) + new-env (assoc env + :recur-point name + :tail? true) + analyzed-bodies (reduce + (partial analyze-fn-body new-env name) + {} + arities)] + (->Fn name + (vals analyzed-bodies) + form + new-env))) + +(defmethod analyze-seq 'try + [env [_ & body :as form]] + (let [analyzed (reduce + (fn [acc f] + (cond + (and (seq? f) + (= (first f) 'catch)) + (let [[_ k ex-nm & body] f] + (assert (keyword? k) "First argument to catch must be a keyword") + (assoc-in acc [:catches k] `(fn [~ex-nm] + ~@body))) + (and (seq? f) + (= (first f) 'finally)) + (let [[_ & body] f] + (assert (nil? (:finally acc)) "Can only have one finally block in a try") + (assoc acc :finally `(fn [] + ~@body))) + + :else (update-in acc [:bodies] conj f))) + {} + body)] + (analyze-form env `(~'pixie.stdlib/-try + (fn [] + ~@(or (:bodies analyzed) + [])) + ~(or (:catches analyzed) + {}) + ~(or (:finally analyzed) + `(fn [] nil)))))) + +(defn add-local [env type bind-name] + (assoc-in env [:locals bind-name] (->Binding type bind-name bind-name env))) + +(defn analyze-fn-body [env fn-name acc [args & body :as form]] + (let [[args variadic?] (let [not& (vec (filter (complement (partial = '&)) args))] + [not& (= '& (last (butlast args)))]) + arity (count args) + arity-idx (if variadic? -1 arity) + new-env (add-local env :fn-self fn-name) + new-env (reduce + (fn [acc arg-name] + (add-local acc :arg arg-name)) + new-env + args)] + (assert (not (acc arity-idx)) (str "Duplicate arity for " (cons args body))) + (assoc acc arity-idx (->FnBody fn-name + arity + args + nil + variadic? + (analyze-form + (assoc new-env :tail? true) + (cons 'do body)) + form + env)))) + +(defmethod analyze-seq 'let* + [env [_ bindings & body :as form]] + (assert (even? (count bindings)) "Let requires an even number of bindings") + (let [parted (partition 2 bindings) + [new-env bindings] (reduce + (fn [[new-env bindings] [name binding-form :as pair]] + (let [binding-ast (->LetBinding name + (analyze-form + (assoc new-env :tail? false) + binding-form) + binding-form + env)] + [(assoc-in new-env [:locals name] binding-ast) + (conj bindings binding-ast)])) + [env []] + parted)] + (->Let bindings + (analyze-form (assoc new-env + :tail? (:tail? env)) + `(do ~@body)) + form + env))) + +(defmethod analyze-seq 'loop* + [env [_ bindings & body]] + (let [parted (partition 2 bindings)] + (analyze-form (assoc env :recur-point '__loop__fn__) + `((fn ~'__loop__fn__ ~(mapv first parted) + ~@body) + ~@(mapv second parted))))) + +(defmethod analyze-seq 'recur + [env [_ & args]] + (assert (:tail? env) "Can only recur at tail position") + (assert (:recur-point env) "Don't have a point to recur to") + (analyze-form env `(~(:recur-point env) ~@args))) + +(defmethod analyze-seq 'var + [env [_ nm]] + (->VarConst @(:ns env) nm nm env)) + +(defmethod analyze-seq 'def + [env [_ nm val :as form]] + (->Def + nm + (analyze-form env val) + form + env)) + +(defmethod analyze-seq 'quote + [env [_ val]] + (if (map? val) + (analyze-seq env + (with-meta + `(pixie.stdlib/hashmap ~@(reduce + (fn [acc [k v]] + (-> acc + (conj `(~'quote k)) + (conj `(~'quote v)))) + [] + val)) + (meta val))) + (->Const val env))) + +(defmethod analyze-seq 'in-ns + [env [_ nsp]] + (let [nsp (if (and (:bootstrap? env) + (= (name nsp) "pixie.bootstrap-macros")) + (symbol "pixie.stdlib") + nsp)] + (reset! (:ns env) (symbol (name nsp))) + (in-ns nsp) + (in-ns :pixie.compiler) + (analyze-form env (list 'pixie.stdlib/-in-ns (keyword (name nsp)))))) + +(defmethod analyze-seq 'local-macro + [env [_ [nm replace] & body :as form]] + (let [new-env (assoc-in env [:locals nm] {:op :local-macro + :name nm + :replace-with replace + :form form})] + (analyze-form new-env (cons 'do body)))) + +(defmethod analyze-form nil + [env _] + (->Const nil env)) + +(defmethod analyze-form :bool + [env form] + (->Const form env)) + +(defn keep-meta [new old] + (if-let [md (meta old)] + (if (satisfies? IMeta new) + (with-meta new md) + new) + new)) + +(defn resolve-sym [env sym] + (cond + (and (:bootstrap? env)) + (try (resolve-in (the-ns :pixie.bootstrap-macros) + (symbol (name sym))) + (catch :pixie.stdlib/AssertionException ex + nil)) + + (namespace sym) + (try (resolve-in (the-ns (namespace sym)) + (symbol (name sym))) + (catch :pixie.stdlib/AssertionException ex + nil)) + + :else + (try (resolve-in (the-ns @(:ns env)) + sym) + (catch :pixie.stdlib/AssertionException ex + nil)))) + +(defmethod analyze-seq :default + [env [sym & args :as form]] + (let [sym (if (and (symbol? sym) + (= "pixie.bootstrap-macros" (namespace sym))) + (symbol (str "pixie.stdlib/" (name sym))) + sym) + resolved (when (symbol? sym) + (resolve-sym env sym))] + (cond + (and (symbol? sym) + (string/starts-with? (name sym) ".-")) + (let [sym-kw (keyword (string/substring (name sym) 2)) + result (analyze-form env (keep-meta `(~'pixie.stdlib/-get-field ~@args ~sym-kw) + form))] + result) + + (and resolved + (macro? @resolved)) + (analyze-form env (keep-meta (apply @resolved args) + form)) + + :else + (->Invoke (let [new-env (assoc env :tail? false)] + (mapv (partial analyze-form new-env) form)) + (:tail? env) + form + env)))) + + +(defmethod analyze-form :number + [env x] + (->Const x env)) + +(defmethod analyze-form :keyword + [env x] + (->Const x env)) + +(defmethod analyze-form :string + [env x] + (->Const x env)) + +(defmethod analyze-form :char + [env x] + (->Const x env)) + +(defmethod analyze-form :seq + [env x] + (analyze-seq env x)) + +(defmethod analyze-form :symbol + [env x] + (if-let [local (get-in env [:locals x])] + (if (= (:op local) :local-macro) + (analyze-form env (:replace-with local)) + (assoc local :form x)) + (maybe-var env x))) + +(defmethod analyze-form :vector + [env x] + (->Vector (mapv (partial analyze-form env) x) + x + env)) + +(defmethod analyze-form :map + [env x] + (analyze-seq env + (with-meta + `(pixie.stdlib/hashmap ~@(reduce + (fn [acc [k v]] + (-> acc + (conj k) + (conj v))) + [] + x)) + (meta x)))) + +(defmethod analyze-form :set + [env x] + (analyze-seq env + (with-meta + `(pixie.stdlib/set ~(vec x)) + (meta x)))) + +(defn maybe-var [env x] + (let [nsp (if (= (namespace x) "pixie.bootstrap-macros") + (symbol (str "pixie.stdlib/" (name x))) + x)] + (->Var @(:ns env) nsp nsp env))) + + +;; ENV Functions + +(defn new-env + "Creates a new (empty) environment" + [bootstrap?] + (->Env + (atom (symbol "pixie.stdlib")) + nil + {} + true + nil + bootstrap? + nil)) + + +(defn analyze + ([form] + (analyze form (new-env false))) + ([form env] + (analyze-form env form))) + + diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index ecfbd5ee..7f499ece 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -1,6 +1,9 @@ (ns pixie.io-blocking - (:require [pixie.streams :as st :refer :all] - [pixie.io.common :as common])) + (:require [pixie.streams :as st :refer :all])) + + +(def DEFAULT-BUFFER-SIZE (* 32 1024)) + (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) @@ -15,6 +18,19 @@ (def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) (def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) +(defn stream-reducer [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc))))) + + (deftype FileStream [fp] IInputStream @@ -36,7 +52,7 @@ (fclose fp)) IReduce (-reduce [this f init] - (common/stream-reducer this f init))) + (stream-reducer this f init))) (defn open-read {:doc "Open a file for reading, returning a IInputStream" @@ -72,6 +88,7 @@ IByteOutputStream (write-byte [this val] (assert (integer? val) "Value must be a int") + (println val) (fputc val fp)) IOutputStream (write [this buffer] @@ -129,7 +146,7 @@ (pclose fp)) IReduce (-reduce [this f init] - (common/stream-reducer this f init))) + (stream-reducer this f init))) (defn popen-read {:doc "Open a file for reading, returning a IInputStream" diff --git a/pixie/io.pxi b/pixie/io.pxi index a37188f6..440eab36 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -13,6 +13,8 @@ (uv/defuvfsfn fs_write [file bufs nbufs offset] :result) (uv/defuvfsfn fs_close [file] :result) +(def DEFAULT-BUFFER-SIZE (* 32 1024)) + (deftype FileStream [fp offset uvbuf] IInputStream (read [this buffer len] @@ -50,7 +52,7 @@ (defn read-line "Read one line from input-stream for each invocation. - nil when all lines have been read" +kl nil when all lines have been read" [input-stream] (let [line-feed (into #{} (map int [\newline \return])) buf (buffer 1)] @@ -88,7 +90,7 @@ (-dispose! [this] (fclose fp))) -(deftype BufferedOutputStream [downstream idx buffer] +(deftype BufferedOutputStream [downstream idx buffer disposed?] IByteOutputStream (write-byte [this val] (pixie.ffi/pack! buffer idx CUInt8 val) @@ -99,6 +101,8 @@ (set-field! this :idx 0))) IDisposable (-dispose! [this] + (assert (not disposed?) "Can't dispose twice!") + (set-field! this :disposed? true) (set-buffer-count! buffer idx) (write downstream buffer)) IFlushableStream @@ -124,7 +128,7 @@ ([downstream] (buffered-output-stream downstream common/DEFAULT-BUFFER-SIZE)) ([downstream size] - (->BufferedOutputStream downstream 0 (buffer size)))) + (->BufferedOutputStream downstream 0 (buffer size) false))) (defn buffered-input-stream ([upstream] @@ -152,24 +156,34 @@ (defn spit "Writes the content to output. Output must be a file or an IOutputStream." - [output content] - (cond - (string? output) - (transduce (map identity) - (-> output - open-write - buffered-output-stream - utf8/utf8-output-stream-rf) - (str content)) - - (satisfies? IOutputStream output) - (transduce (map identity) - (-> output - buffered-output-stream - utf8/utf8-output-stream-rf) - (str content)) - - :else (throw [::Exception "Expected a string or IOutputStream"]))) + ([output content] + (spit output content false)) + ([output content dispose?] + (cond + (string? output) + (transduce (map identity) + (-> output + open-write + buffered-output-stream + (utf8/utf8-output-stream-rf dispose?)) + (str content)) + + (satisfies? IOutputStream output) + (transduce (map identity) + (-> output + buffered-output-stream + (utf8/utf8-output-stream-rf dispose?)) + (str content)) + + (satisfies? IByteOutputStream output) + (transduce (map identity) + (-> output + (utf8/utf8-output-stream-rf dispose?)) + (str content)) + + + + :else (throw [::Exception "Expected a string or IOutputStream"])))) (defn slurp "Reads in the contents of input. Input must be a filename or an IInputStream" diff --git a/pixie/main.pxi b/pixie/main.pxi new file mode 100644 index 00000000..82958500 --- /dev/null +++ b/pixie/main.pxi @@ -0,0 +1,116 @@ +(ns pixie.main + (:require [pixie.io-blocking :as io] + [pixie.reader :as reader] + [pixie.compiler :as compiler] + [pixie.ast-output :as ast-out] + [pixie.string :as string])) + +(defmulti print-stack-at :type) +(defmethod print-stack-at :handler + [{:keys [handler]}] + (println "Handler:" (type handler) ":" handler)) + +(defmethod print-stack-at :default + [{:keys [ast type]}] + (if ast + (let [meta (:meta (describe-internal-object ast))] + (println (:line meta) + "in" + (:file meta) + " " + type + "at" + (str (:line-number meta) + ":" + (:column-number meta))) + (println (apply str (repeat (- (:column-number meta) 2) " ")) + "^")) + (println "Unknown location in " type))) + +(defn print-stack-trace [ks] + (doseq [slice ks] + (doseq [k (:slice (describe-internal-object slice))] + (print-stack-at k)))) + +(defn load-file [ns-sym] + (let [name (if (string? ns-sym) + ns-sym + (string/replace (name ns-sym) "." "/")) + name (if (string/ends-with? name ".pxi") + name + (str name ".pxi")) + full-name (reduce + (fn [_ load-path] + (let [nm (str load-path "/" name)] + (println "Looking for " nm) + (when (pixie.io-blocking/file-exists? nm) + (reduced nm)))) + nil + @load-paths)] + (assert full-name (str "Couldn't load file " ns-sym)) + (load-resolved-file full-name))) + +(defn load-resolved-file [full-name] + (let [data (io/slurp full-name) + rdr (reader/metadata-reader + (reader/indexed-reader data) + full-name)] + (with-handler [_ dynamic-var-handler] + (binding [reader/*current-ns* 'user] + (loop [] + (let [d (reader/read rdr false)] + (if (not (= d :eof)) + (let [analyzed (ast-out/to-ast (compiler/analyze d))] + (pixie.ast.internal/eval analyzed) + (recur))))))))) + +(try + (load-file :pixie.ffi-infer) + (catch :* data + (println "ERROR Compiling file" data) + (print-stack-trace (:ks data)))) + + +(def libedit (ffi-library "libedit.dylib")) +(println libedit) +(def -readline (ffi-fn libedit "readline" [CCharP] CCharP)) + +(defrecord Foo [x y]) + +(let [env (compiler/new-env false) + _ (println "MAKE ENV " (:ns env)) + data (reader/read-string "( * 1 2)") + _ (println "got " data) + ast (ast-out/to-ast (compiler/analyze data env))] + (println "_> ast -> " ast) + (println (pixie.ast.internal/eval ast))) + +(defn repl [] + (with-handler [_ dynamic-var-handler] + (binding [reader/*current-ns* 'user] + (let [read-fn #(str (-readline "=>") "\n") + rdr (reader/metadata-reader + (reader/user-space-reader read-fn) + "")] + (loop [] + (let [d (reader/read rdr false)] + (println "got " d) + (println "analyzing ") + (let [analyzed (try + (ast-out/to-ast (compiler/analyze d)) + (catch :* data + (println "ERROR Analyzing" data) + (print-stack-trace (:ks data))))] + (println "analyzed " analyzed) + (try + (println "result :" (pixie.ast.internal/eval analyzed)) + (catch :* data + (println "ERROR Compiling file" data) + (print-stack-trace (:ks data))))) + (if (not (= d :exit-repl)) + (recur)))))))) +(repl) + +(println "done") + +((fn [x] (if (< x 1000) (recur (inc x)))) 0) diff --git a/pixie/pxic-writer.pxi b/pixie/pxic-writer.pxi new file mode 100644 index 00000000..2c030b75 --- /dev/null +++ b/pixie/pxic-writer.pxi @@ -0,0 +1,319 @@ +(ns pixie.pxic-writer + (:require [pixie.ast :as ast] + [pixie.streams :refer [write-byte]] + [pixie.streams.utf8 :refer [utf8-output-stream-rf]] + [pixie.io :refer [spit]])) + +(def MAX_INT32 (bit-shift-left 1 31)) + +(defmacro defenum [nms]) + + + + +(defmacro defbytecodes [bytecodes] + `(do ~@(map + (fn [idx k] + `(def ~(symbol (name k)) ~idx)) + (range) + bytecodes))) + +(defbytecodes + [:CACHED-OBJECT + :INT + :FLOAT + :INT-STRING + :STRING + :TRUE + :FALSE + :NIL + :VAR + :KEYWORD + :SYMBOL + :NEW-CACHED-OBJECT + :DO + :INVOKE + :VAR + :CONST + :FN + :LOOKUP + :IF + :LET + :META + :LINE-META + :VAR-CONST + :CHAR + :VECTOR + :RECUR]) + +(def *cache* nil) +(set-dynamic! (var *cache*)) + +(def *old-meta* nil) +(set-dynamic! (var *old-meta*)) + + +(defprotocol IWriterCache + (write-cached-obj [this val wfn])) + +(deftype NullWriterCache [] + IWriterCache + (write-cached-obj [this val wfn] + (apply wfn val))) + +(deftype WriterCache [os cache] + IWriterCache + (write-cached-obj [this val wfn] + (if-let [idx (cache val)] + (do (write-tag os CACHED-OBJECT) + (write-int-raw os idx)) + (let [idx (count cache)] + (set-field! this :cache (assoc cache val idx)) + (write-tag os NEW-CACHED-OBJECT) + (apply wfn val))))) + + +(defn writer-cache [os] + (->WriterCache os {})) + +(defn write-raw-string [os s] + (assert (string? s) (str "Expected String, got " str)) + (write-int-raw os (count s)) + (spit os s false)) + +(defn write-int-raw [os i] + (if (<= 0 i MAX_INT32) + (do (write-byte os (bit-and i 0xFF)) + (write-byte os (bit-and (bit-shift-right i 8) 0xFF)) + (write-byte os (bit-and (bit-shift-right i 16) 0xFF)) + (write-byte os (bit-and (bit-shift-right i 24) 0xFF))) + (throw [:pixie.pxic-writer/OversizedInt + "Collection sizes are limited to 32 bits in pxic files"]))) + +(defn write-int [os i] + (if (<= 0 i MAX_INT32) + (do (write-byte os INT) + (write-int-raw os i)) + (do (write-byte os INT-STRING) + (write-raw-string os (str i))))) + +(defn write-meta [os ast] + (let [m (or (meta (:form ast)) + (:meta (:env ast)) + *old-meta*)] + (if (and m (:file m)) + (write-object os (ast/->Meta + (ast/->LineMeta + (str (:line m)) + (:file m) + (:line-number m)) + (:column-number m))) + (write-object os nil)))) + +(defn write-tag [os tag] + (write-byte os tag)) + +(defprotocol IPxicObject + (-write-object [this os])) + +(defmulti write-ast (fn [os ast] + (:op ast))) + + + + + + + + + + + +(extend-protocol IPxicObject + + ast/If + (-write-object [{:keys [test then else] :as ast} os] + (write-tag os IF) + (write-object os test) + (write-object os then) + (write-object os else) + (write-meta os ast)) + + ast/Let + (-write-object [{:keys [bindings body] :as ast} os] + (write-tag os LET) + (write-int-raw os (count bindings)) + (doseq [{:keys [name value]} bindings] + (write-object os (keyword (pixie.stdlib/name name))) + (write-object os value)) + (write-object os body) + (write-meta os ast)) + + + ast/LetBinding + (-write-object + [{:keys [name] :as ast} os] + (write-tag os LOOKUP) + (write-object os (keyword (pixie.stdlib/name name))) + (write-meta os ast)) + + + ast/Binding + (-write-object + [{:keys [name] :as ast} os] + (write-tag os LOOKUP) + (write-object os (keyword (pixie.stdlib/name name))) + (write-meta os ast)) + + + ast/FnBody + (-write-object [{:keys [name args closed-overs body] :as ast} os] + (write-tag os FN) + (write-raw-string os (str name)) + (write-int-raw os (count args)) + (doseq [arg args] + (write-object os (keyword (str arg)))) + (write-int-raw os (count closed-overs)) + (doseq [co closed-overs] + (write-object os (keyword (str co)))) + (write-object os body) + (write-meta os ast)) + + + + ast/Meta + (-write-object [{:keys [line column-number]} os] + (write-tag os META) + (write-object os line) + (write-int-raw os column-number)) + + ast/LineMeta + (-write-object [ast os] + (write-cached-obj *cache* + [:line-meta ast] + (fn [_ {:keys [line file line-number]}] + (write-tag os LINE-META) + (write-raw-string os file) + (write-raw-string os line) + (write-int-raw os line-number)))) + + + + ast/Const + (-write-object [{:keys [form]} os] + (write-tag os CONST) + (write-object os form)) + + + ast/VarConst + (-write-object [{:keys [ns name] :as ast} os] + (write-tag os VAR-CONST) + (write-raw-string os (str ns)) + (write-object os name) + (write-meta os ast)) + + ast/Var + (-write-object [{:keys [ns var-name] :as ast} os] + (write-tag os VAR) + (write-raw-string os (str ns)) + (write-object os var-name) + (write-meta os ast)) + + ast/Invoke + (-write-object [{:keys [args env] :as ast} os] + (write-tag os INVOKE) + (write-int-raw os (count args)) + (doseq [arg args] + (write-object os arg)) + (write-meta os ast)) + + ast/Recur + (-write-object [{:keys [args env] :as ast} os] + (write-tag os RECUR) + (write-int-raw os (count args)) + (doseq [arg args] + (write-object os arg)) + (write-meta os ast)) + + ast/Do + (-write-object [{:keys [statements ret] :as ast} os] + (write-tag os DO) + (write-int-raw os (inc (count statements))) + (doseq [statement statements] + (write-object os statement)) + (write-object os ret) + (write-meta os ast)) + + IVector + (-write-object [this os] + (write-tag os VECTOR) + (write-int-raw os (count this)) + (doseq [item this] + (write-object os item))) + + String + (-write-object [this os] + (write-cached-obj *cache* + [:string this] + (fn [_ this] + (write-tag os STRING) + (write-raw-string os this)))) + + Keyword + (-write-object [this os] + (write-cached-obj *cache* + [:keyword this] + (fn [_ this] + (write-tag os KEYWORD) + (if (namespace this) + (write-raw-string os (str (namespace this) "/" (name this))) + (write-raw-string os (name this)))))) + + Symbol + (-write-object [this os] + (write-cached-obj *cache* + [:symbol this] + (fn [_ this] + (write-tag os SYMBOL) + (if (namespace this) + (write-raw-string os (str (namespace this) "/" (name this))) + (write-raw-string os (name this)))))) + + Integer + (-write-object [this os] + (write-int os this)) + + Bool + (-write-object [this os] + (if this + (write-tag os TRUE) + (write-tag os FALSE))) + + Nil + (-write-object [this os] + (write-tag os NIL)) + + Character + (-write-object [this os] + (write-tag os CHAR) + (write-raw-string os (str this))) + + Object + (-write-object [this os] + (throw [:pixie.stdlib/IllegalArgumentException + (str "Can't write " this)]))) + +(defn simplify [ast] + (let [simplified (ast/simplify-ast ast)] + (if (identical? simplified ast) + ast + (recur simplified)))) + +(defn write-object [os ast] + + (binding [*old-meta* (or (and + (satisfies? ast/IAst ast) + (or (meta (:form ast)) + (:meta (:env ast)))) + *old-meta*)] + (-write-object (simplify ast) os))) diff --git a/pixie/reader.pxi b/pixie/reader.pxi new file mode 100644 index 00000000..6cf26ad0 --- /dev/null +++ b/pixie/reader.pxi @@ -0,0 +1,376 @@ +(ns pixie.reader + (:require [pixie.string :as string])) + +(def *current-ns* nil) +(set-dynamic! (var *current-ns*)) + + +(defprotocol IPushbackReader + (read-ch [this]) + (unread-ch [this])) + +(defprotocol IMetadataReader + (metadata [this])) + +(deftype IndexedReader [s idx] + IPushbackReader + (read-ch [this] + (if (>= idx (count s)) + :eof + (let [ch (nth s idx)] + (set-field! this :idx (inc idx)) + ch))) + (unread-ch [this] + (set-field! this :idx (dec idx)))) + +(defn indexed-reader [s] + (->IndexedReader s 0)) + +(deftype UserSpaceReader [string-rdr reader-fn] + IPushbackReader + (read-ch [this] + (when-not string-rdr + (let [result (reader-fn)] + (if (eof? result) + (set-field! this :string-rdr :eof) + (set-field! this :string-rdr (->IndexedReader result 0))))) + + (if (eof? string-rdr) + :eof + (let [v (read-ch string-rdr)] + (if (eof? v) + (do (set-field! this :string-rdr nil) + (read-ch this)) + v)))) + (unread-ch [this] + (unread-ch string-rdr))) + +(defn user-space-reader [f] + (->UserSpaceReader nil f)) + + +(deftype MetaDataReader [parent-reader line-number column-number line + prev-line-number prev-column-number prev-line prev-chr + filename has-unread cur-chr] + IPushbackReader + (read-ch [this] + (if has-unread + (do (set-field! this :has-unread false) + prev-chr) + (let [ch (read-ch parent-reader)] + (set-field! this :prev-column-number column-number) + (set-field! this :prev-line-number line-number) + (set-field! this :prev-line prev-line) + (when (string? @line) + (set-field! this :line (atom []))) + (if (identical? ch \n) + (do (swap! line (fn [x] (apply str x))) + (set-field! this :line-number (inc line-number)) + (set-field! this :column-number 0)) + (do (swap! line conj ch) + (set-field! this :column-number (inc column-number)))) + + (set-field! this :cur-chr ch) + ch))) + + (unread-ch [this] + (assert (not has-unread) "Can't unread twice") + (set-field! this :has-unread true) + (set-field! this :prev-chr cur-chr)) + + IMetadataReader + (metadata [this] + {:line line + :line-number line-number + :column-number column-number + :file filename})) + +(defn metadata-reader [parent file] + (->MetaDataReader parent 1 0 (atom []) 1 0 nil \0 file false \0)) + + +(def whitespace? (contains-table \return \newline \, \space \tab)) + +(def digit? (apply contains-table "0123456789")) + +(defn eof? [x] + (identical? x :eof)) + +(defn terminating-macro? [ch] + (and (not= ch \#) + (not= ch \') + (not= ch \%) + (handlers ch))) + +(defn eat-whitespace [rdr] + (let [ch (read-ch rdr)] + (if (whitespace? ch) + (eat-whitespace rdr) + ch))) + +(defn assert-not-eof [ch msg] + (assert (not (identical? ch :eof)) msg)) + +(defn make-coll-reader [build-fn open-ch close-ch] + (fn rdr-internal + ([rdr] + (rdr-internal rdr [])) + ([rdr coll] + (let [ch (eat-whitespace rdr)] + #_(assert-not-eof ch (str "Unmatched delimiter " open-ch)) + (if (identical? ch close-ch) + (apply build-fn coll) + (let [_ (unread-ch rdr) + itm (read-inner rdr true false)] + (if (identical? itm rdr) + (rdr-internal rdr coll) + (rdr-internal rdr (conj coll itm))))))))) + + +(defn make-unmatched-handler [unhandled-ch] + (fn [rdr] + (throw [:pixie.reader/ParseError + (str "Unmatched delimiter " unhandled-ch)]))) + + +(def *gen-sym-env* nil) +(set-dynamic! (var *gen-sym-env*)) + +(defn syntax-quote-reader [rdr] + (let [form (read-inner rdr true)] + (binding [*gen-sym-env* {}] + (syntax-quote form)))) + +(defn unquote? [form] + (and (seq? form) + (= (first form) 'unquote))) + +(defn unquote-splicing? [form] + (and (seq? form) + (= (first form) 'unquote-splicing))) + +(defn syntax-quote [form] + (cond + (and (symbol? form) + nil ; TODO: Compiler special + ) + (list 'quote form) + + (symbol? form) + (cond + (namespace form) + (list 'quote form) + + (string/ends-with? (name form) "#") + (let [gmap *gen-sym-env* + _ (assert gmap "Gensym literal used outside of a syntax quote") + gs (or (get gmap form) + (let [s (gensym (str (name form) "__"))] + (set! *gen-sym-env* (assoc gmap form s)) + s))] + (list 'quote gs)) + + :else + (list 'quote (symbol (str (name *current-ns*) "/" (name form))))) + + (unquote? form) + (first (next form)) + + (unquote-splicing? form) + (assert false "Unquote splicing used outside a list") + + (vector? form) + (list 'pixie.stdlib/apply 'pixie.apply/concat (expand-list form)) + + (and form (seq? form)) + (list 'pixie.stdlib/apply 'pixie.stdlib/list (expand-list form)) + + :else (list 'quote form))) + +(defn expand-list [form] + (reduce + (fn [acc itm] + (cond + (unquote? itm) + (conj acc [(nth form 1)]) + + (unquote-splicing? form) + (conj acc (nth form 1)) + + :else (conj acc (syntax-quote itm)))) + [] + form)) + +(defn deref-reader [rdr] + (list 'pixie.stdlib/deref (read-inner rdr true))) + + +(defn skip-line-reader [rdr] + (if (identical? \newline (read-ch rdr)) + rdr + (skip-line-reader rdr))) + +(defn meta-reader [rdr] + (let [m (read-inner rdr true) + o (read-inner rdr true) + m (cond + (keyword? m) {m true} + (symbol? m) {:tag m} + :else m)] + (if (has-meta? o) + (with-meta o m) + o))) + +(defn unquote-reader [rdr] + (let [ch (read-ch rdr) + sym (if (identical? ch \@) + 'pixie.stdlib/unquote-splicing + (do (unread-ch rdr) + 'pixie.stdlib/unquote)) + form (read-inner rdr true)] + (list sym form))) + +(def string-literals + (switch-table + \" \" + \\ \\ + \n \newline + \r \return + \t \tab)) + +(defn literal-string-reader [rdr] + (let [sb (string-builder) + sb-fn (fn [x] + (-add-to-string-builder sb x))] + (loop [] + (let [ch (read-ch rdr)] + (cond + (eof? ch) (throw [:pixie.reader/ParseError + "Unmatched string quote"]) + (identical? \" ch) (str sb) + + (identical? \\ ch) (let [v (read-ch rdr) + _ (when (eof? v) + (throw [:pixie.reader/ParseError + "End of file after escape character"])) + converted (string-literals v)] + (if converted + (do (sb-fn converted) + (recur)) + (throw [:pixie.reader/ParseError + (str "Unhandled escape character " v)]))) + :else (do (sb-fn ch) + (recur))))))) + +(defn keyword-reader [rdr] + (let [ch (read-ch rdr)] + (assert (not= \: ch)) + (let [itm (read-symbol rdr ch)] + (if (namespace itm) + (keyword (str (namespace itm) "/" (name itm))) + (keyword (name itm)))))) + +(defn quote-reader [rdr] + (list 'quote (read-inner rdr true))) + +(def handlers + (switch-table + \( (make-coll-reader list \( \)) + \[ (make-coll-reader vector \[ \]) + \{ (make-coll-reader hashmap \{ \}) + \] (make-unmatched-handler \]) + \) (make-unmatched-handler \)) + \} (make-unmatched-handler \}) + \: keyword-reader + \` syntax-quote-reader + \@ deref-reader + \; skip-line-reader + \^ meta-reader + \~ unquote-reader + \" literal-string-reader + \' quote-reader)) + +(defn read-number [rdr ch] + (let [sb (string-builder) + sb-fn (fn [x] + (-add-to-string-builder sb x))] + (-str ch sb-fn) + (loop [sb-fn sb-fn] + (let [ch (read-ch rdr)] + (if (or (whitespace? ch) + (terminating-macro? ch) + (eof? ch)) + + (let [val (-parse-number (str sb))] + (unread-ch rdr) + (if val + val + (symbol val))) + (do (-str ch sb-fn) + (recur sb-fn))))))) + + +(defn read-symbol [rdr ch] + (let [sb (string-builder) + sb-fn (fn [x] + (-add-to-string-builder sb x))] + (-str ch sb-fn) + (loop [sb-fn sb-fn] + (let [ch (read-ch rdr)] + (if (or (whitespace? ch) + (terminating-macro? ch) + (eof? ch)) + + (let [val (interpret-symbol (str sb))] + (unread-ch rdr) + val) + (do (-str ch sb-fn) + (recur sb-fn))))))) + +(defn interpret-symbol [s] + (cond + (= s "true") true + (= s "false") false + (= s "nil") nil + :else (symbol s))) + + +(defn read-inner + ([rdr error-on-eof] + (read-inner rdr error-on-eof true)) + ([rdr error-on-eof always-return-form] + (let [ch (eat-whitespace rdr)] + (if (identical? ch :eof) + (if error-on-eof + (assert-not-eof ch "Unexpeced EOF while reading") + ch) + (let [m (when (satisfies? IMetadataReader rdr) + (metadata rdr)) + macro (handlers ch) + itm (cond + macro (let [itm (macro rdr)] + (if (and always-return-form + (identical? itm rdr)) + (read-inner rdr error-on-eof always-return-form) + itm)) + (digit? ch) (read-number rdr ch) + (identical? ch \-) (let [ch2 (read-ch rdr)] + (if (digit? ch2) + (do (unread-ch rdr) + (read-number rdr ch)) + (do (unread-ch rdr) + (read-symbol rdr ch)))) + :else (read-symbol rdr ch))] + (if (identical? itm rdr) + itm + (if (has-meta? itm) + (with-meta itm m) + itm))))))) + +(defn read [rdr error-on-eof] + (read-inner rdr error-on-eof)) + + +(defn read-string [s] + (read (->IndexedReader s 0) + false)) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 2c3b9904..02aecef0 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -772,6 +772,7 @@ there's a value associated with the key. Use `some` for checking for values." :examples [["((comp inc first) [41 2 3])" nil 42]] :signatures [[f] [f & fs]] :added "0.1"} + ([] identity) ([f] f) ([f1 f2] (fn [& args] @@ -1252,47 +1253,6 @@ Creates new maps if the keys are not present." `(do ~type-decl ~ctor ~@proto-bodies))) - -(defmacro defrecord - {:doc "Define a record type. - -Similar to `deftype`, but supports construction from a map using `map->Type` -and implements IAssociative, ILookup and IObject." - :added "0.1"} - [nm fields & body] - (let [ctor-name (symbol (str "->" (name nm))) - map-ctor-name (symbol (str "map" (name ctor-name))) - fields (transduce (map (comp keyword name)) conj fields) - type-from-map `(defn ~map-ctor-name [m] - (apply ~ctor-name (map #(get m %) ~fields))) - default-bodies ['IAssociative - `(-assoc [self k v] - (let [m (reduce #(assoc %1 %2 (. self %2)) {} ~fields)] - (~map-ctor-name (assoc m k v)))) - `(-contains-key [self k] - (contains? ~(set fields) k)) - `(-dissoc [self k] - (throw [:pixie.stdlib/NotImplementedException - "dissoc is not supported on defrecords"])) - 'ILookup - `(-val-at [self k not-found] - (if (contains? ~(set fields) k) - (. self k) - not-found)) - 'IObject - `(-str [self] - (str "<" ~(name nm) " " (reduce #(assoc %1 %2 (. self %2)) {} ~fields) ">")) - `(-eq [self other] - (and (instance? ~nm other) - ~@(map (fn [field] - `(= (. self ~field) (. other ~field))) - fields))) - `(-hash [self] - (throw [:pixie.stdlib/NotImplementedException "not implemented"]))] - deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] - `(do ~type-from-map - ~deftype-decl))) - (defn print {:doc "Prints the arguments, seperated by spaces." :added "0.1"} @@ -1327,11 +1287,7 @@ and implements IAssociative, ILookup and IObject." (puts (apply pr-str args)) nil) -(defn repeat - ([x] - (cons x (lazy-seq* (fn [] (repeat x))))) - ([n x] - (take n (repeat x)))) + (defn repeatedly {:doc "Returns a lazy seq that contains the return values of repeated calls to f. @@ -1351,11 +1307,12 @@ and implements IAssociative, ILookup and IObject." (assert (= (count binding) 2) "expected a binding and a collection") (let [b (first binding) s (second binding)] - `(loop [s# (seq ~s)] - (if s# - (let [~b (first s#)] - ~@body - (recur (next s#))))))) + `(reduce + (fn [acc# ~b] + ~@body + nil) + nil + ~s))) (defmacro doc {:doc "Returns the documentation of the given value." @@ -1479,19 +1436,7 @@ The new value is thus `(apply f current-value-of-atom args)`." ([x y & more] `(let [r# ~x] (if r# r# (or ~y ~@more))))) -(defmacro when [test & body] - `(if ~test (do ~@body))) -(defmacro when-not [test & body] - `(if (not ~test) (do ~@body))) - -(defmacro when-let [binding & body] - (let [bind (nth binding 0 nil) - test (nth binding 1 nil)] - `(let [tmp# ~test] - (when tmp# - (let [~bind tmp#] - ~@body))))) (defmacro if-not ([test then] `(if-not ~test ~then nil)) @@ -1835,6 +1780,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (and (< step 0) (> start stop))) (cons start (lazy-seq* #(range (+ start step) stop step)))))) + (extend -str Range (fn [v] (-str (seq v)))) @@ -2069,15 +2015,22 @@ The params can be destructuring bindings, see `(doc let)` for details."} `(fn* ~@name ~(first (first decls)) ~@(next (first decls))) `(fn* ~@name ~@decls)))) -(deftype MultiMethod [dispatch-fn default-val methods] +(deftype MultiMethod [name dispatch-fn default-val methods] IFn (-invoke [self & args] (let [dispatch-val (apply dispatch-fn args) method (if (contains? @methods dispatch-val) (get @methods dispatch-val) (get @methods default-val)) - _ (assert method (str "no method defined for " dispatch-val))] - (apply method args)))) + _ (assert method (str "no method defined for " dispatch-val " on " name))] + (try + (apply method args) + (catch ex + (throw (add-exception-info ex (str "In multimethod " + name + " dispatching on " + dispatch-val + "\n") args))))))) (defmacro defmulti {:doc "Define a multimethod, which dispatches to its methods based on dispatch-fn." @@ -2095,8 +2048,10 @@ The params can be destructuring bindings, see `(doc let)` for details."} [(merge meta (first args)) (next args)] [meta args]) dispatch-fn (first args) - options (apply hashmap (next args))] - `(def ~name (->MultiMethod ~dispatch-fn ~(get options :default :default) (atom {}))))) + options (apply hashmap (next args)) + result `(def ~name (->MultiMethod ~(str name) ~dispatch-fn ~(get options :default :default) (atom {})))] + (println result) + result)) (defmacro defmethod {:doc "Define a method of a multimethod. See `(doc defmulti)` for details." @@ -2336,7 +2291,7 @@ Calling this function on something that is not ISeqable returns a seq with that (defn mapv ([f col] - (transduce (map f) conj col))) + (transduce (map f) conj! col))) (defn macroexpand-1 {:doc "If form is a macro call, returns the expanded form. Does nothing if not a macro call." @@ -2450,7 +2405,78 @@ Calling this function on something that is not ISeqable returns a seq with that (let ~(vec (interleave bindings binding-syms)) ~@body))))) + + +(extend -transient PersistentHashMap + (fn [m] + (reduce + (fn [acc [k v]] + (add-to-dict-map acc k v)) + (new-dict-map) + m))) + +(extend -persistent! PersistentHashMap identity) + + (extend -str Namespace (fn [v] (str ""))) (extend -repr Namespace -str) + + +(defn -make-record-assoc-body [cname fields] + (let [k-sym (gensym "k") + v-sym (gensym "v") + this-sym (gensym "this") + result `(-assoc [~this-sym ~k-sym ~v-sym] + (case ~k-sym + ~@(mapcat + (fn [k] + [k `(~cname ~@(mapv (fn [x] + (if (= x k) + v-sym + `(get-field ~this-sym ~x))) + fields))]) + fields) + (throw [:pixie.stdlib/NotImplementedException + (str "Can't assoc to a unknown field: " ~k-sym)])))] + result)) + +(defmacro defrecord + {:doc "Define a record type. + +Similar to `deftype`, but supports construction from a map using `map->Type` +and implements IAssociative, ILookup and IObject." + :added "0.1"} + [nm field-syms & body] + (let [ctor-name (symbol (str "->" (name nm))) + map-ctor-name (symbol (str "map" (name ctor-name))) + fields (transduce (map (comp keyword name)) conj field-syms) + type-from-map `(defn ~map-ctor-name [m] + (apply ~ctor-name (map #(get m %) ~fields))) + default-bodies ['IAssociative + (-make-record-assoc-body ctor-name fields) + `(-contains-key [self k] + (contains? ~(set fields) k)) + `(-dissoc [self k] + (throw [:pixie.stdlib/NotImplementedException + "dissoc is not supported on defrecords"])) + 'ILookup + `(-val-at [self k not-found] + (if (contains? ~(set fields) k) + (get-field self k) + not-found)) + 'IObject + `(-str [self] + (str "<" ~(name nm) " " (reduce #(assoc %1 %2 (. self %2)) {} ~fields) ">")) + `(-eq [self other] + (and (instance? ~nm other) + ~@(map (fn [field] + `(= (. self ~field) (. other ~field))) + fields))) + `(-hash [self] + (hash [~@field-syms]))] + deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] + `(do ~type-from-map + ~deftype-decl))) + diff --git a/pixie/streams/utf8.pxi b/pixie/streams/utf8.pxi index bc9b4044..9d49eb73 100644 --- a/pixie/streams/utf8.pxi +++ b/pixie/streams/utf8.pxi @@ -27,7 +27,6 @@ (-dispose! [this] (dispose! out))) - (deftype UTF8InputStream [in bad-char] IUTF8InputStream (read-char [this] @@ -68,11 +67,15 @@ [o] (->UTF8OutputStream o)) -(defn utf8-output-stream-rf [output-stream] - (let [fp (utf8-output-stream output-stream)] - (fn ([] 0) - ([_] (dispose! fp)) - ([_ chr] - (assert (char? chr)) - (write-char fp chr) - nil)))) +(defn utf8-output-stream-rf + ([output-stream] + (utf8-output-stream-rf output-stream true)) + ([output-stream dispose?] + (let [fp (utf8-output-stream output-stream)] + (fn ([] 0) + ([_] (when dispose? + (dispose! fp))) + ([_ chr] + (assert (char? chr)) + (write-char fp chr) + nil))))) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 64ad57c9..4161944d 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -2,7 +2,7 @@ import pixie.vm.object as object from pixie.vm.object import affirm, runtime_error import pixie.vm.code as code -from pixie.vm.code import as_var +from pixie.vm.code import as_var, extend from pixie.vm.primitives import nil, true, false import pixie.vm.numbers as numbers from pixie.vm.cons import cons @@ -17,6 +17,7 @@ from pixie.vm.persistent_hash_set import EMPTY as EMPTY_SET from pixie.vm.persistent_list import EmptyList import pixie.vm.compiler as compiler +import pixie.vm.stdlib as proto from rpython.rlib.rbigint import rbigint from rpython.rlib.rsre import rsre_re as re @@ -154,6 +155,9 @@ def __repr__(self): return self._str return u"".join(self._chrs) +@extend(proto._str, LinePromise) +def _str(x): + return rt.wrap(x.__repr__()) class MetaDataReader(PlatformReader): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 5b4326a0..cf1f41f8 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -336,7 +336,7 @@ def _satisfies(proto, o): @as_var("read-string") def _read_string(s): import pixie.vm.reader as reader - return reader.read(reader.StringReader(unicode(rt.name(s))), True) + return reader.read(reader.MetaDataReader(reader.StringReader(unicode(rt.name(s)))), True) # XXX seems broken under jit. @as_var("eval") @@ -723,7 +723,7 @@ def _var(ns, nm): @as_var("set-dynamic!") def set_dynamic(var): - affirm(isinstance(var, Var), u"set-dynamic! expects a var as an argument") + affirm(isinstance(var, Var), u"dynamic! expects a var as an argument") var.set_dynamic() return var @@ -913,3 +913,5 @@ def _add_exception_info(ex, str, data): assert isinstance(ex, RuntimeException) ex._trace.append(ExtraCodeInfo(rt.name(str), data)) return ex + + diff --git a/pixie/vm/string.py b/pixie/vm/string.py index c9be8844..a8358d04 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -152,3 +152,4 @@ def _namespace(self): def _hash(self): assert isinstance(self, String) return rt.wrap(intmask(util.hash_unencoded_chars(self._str))) + diff --git a/pixie/vm2/__init__.py b/pixie/vm2/__init__.py new file mode 100644 index 00000000..f87606f0 --- /dev/null +++ b/pixie/vm2/__init__.py @@ -0,0 +1 @@ +__author__ = 'tim' diff --git a/pixie/vm2/array.py b/pixie/vm2/array.py new file mode 100644 index 00000000..9647ec3c --- /dev/null +++ b/pixie/vm2/array.py @@ -0,0 +1,273 @@ +import pixie.vm2.rt as rt +import pixie.vm2.object as object +from pixie.vm2.object import affirm +from pixie.vm2.code import extend, as_var +from pixie.vm2.numbers import Integer +from pixie.vm2.primitives import nil +#import pixie.vm.stdlib as proto +import rpython.rlib.jit as jit +from rpython.rtyper.lltypesystem import lltype +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.objectmodel import we_are_translated +from pixie.vm2.keyword import keyword +import pixie.vm2.rt as rt +UNROLL_IF_SMALLER_THAN = 8 + +KW_count = keyword(u"count") + + +class Array(object.Object): + _type = object.Type(u"pixie.stdlib.Array") + _immutable_fields_ = ["_list"] + def type(self): + return Array._type + + def __init__(self, lst): + if we_are_translated(): + for x in lst: + assert x is not None + self._list = lst + + @jit.unroll_safe + def reduce_small(self, f, init): + for x in range(len(self._list)): + if rt.reduced_QMARK_(init): + return rt.deref(init) + init = f.invoke([init, self._list[x]]) + return init + + + def reduce_large(self, f, init): + for x in range(len(self._list)): + if rt.reduced_QMARK_(init): + return rt.deref(init) + init = f.invoke([init, self._list[x]]) + return init + + def get_field(self, k, not_found): + if k is KW_count: + return rt.wrap(len(self._list)) + return not_found + + def array_val(self): + return self._list + +@as_var("array") +def array__args(lst): + return Array(lst) + +@as_var("make-array") +def _make_array(size): + return Array([nil] * size.int_val()) + +#@extend(proto._count, Array) +#def _count(self): +# assert isinstance(self, Array) +# return rt.wrap(len(self._list)) +# +# @extend(proto._nth, Array) +# def _nth(self, idx): +# assert isinstance(self, Array) +# ival = idx.int_val() +# if ival < len(self._list): +# return self._list[ival] +# else: +# affirm(False, u"Index out of Range") +# +# @extend(proto._nth_not_found, Array) +# def _nth_not_found(self, idx, not_found): +# assert isinstance(self, Array) +# ival = idx.int_val() +# if ival < len(self._list): +# return self._list[ival] +# else: +# return not_found +# +# @extend(proto._reduce, Array) +# def reduce(self, f, init): +# assert isinstance(self, Array) +# if len(self._list) > UNROLL_IF_SMALLER_THAN: +# return self.reduce_large(f, init) +# return self.reduce_small(f, init) +# +# @extend(proto._seq, Array) +# def _seq(self): +# assert isinstance(self, Array) +# if rt.count(self) > 0: +# return ArraySeq(0, self) +# else: +# return nil +# +# class ArraySeq(object.Object): +# _type = object.Type(u"pixie.stdlib.ArraySeq") +# _immutable_fields_ = ["_idx", "_w_array"] +# +# def __init__(self, idx, array): +# self._idx = idx +# self._w_array = array +# +# def first(self): +# return rt.nth(self._w_array, rt.wrap(self._idx)) +# +# def next(self): +# if self._idx < rt.count(self._w_array) - 1: +# return ArraySeq(self._idx + 1, self._w_array) +# else: +# return nil +# +# def reduce(self, f, init): +# for x in range(self._idx, rt.count(self._w_array)): +# if rt.reduced_QMARK_(init): +# return rt.deref(init) +# init = f.invoke([init, rt.nth(self._w_array, rt.wrap(x))]) +# return init +# +# def type(self): +# return self._type +# # +# @extend(proto._first, ArraySeq) +# def _first(self): +# assert isinstance(self, ArraySeq) +# return self.first() +# +# @extend(proto._next, ArraySeq) +# def _next(self): +# assert isinstance(self, ArraySeq) +# return self.next() +# +# @extend(proto._seq, ArraySeq) +# def _seq(self): +# assert isinstance(self, ArraySeq) +# return self +# +# @extend(proto._reduce, ArraySeq) +# def _reduce(self, f, init): +# assert isinstance(self, ArraySeq) +# return self.reduce(f, init) +# +# def array(lst): +# assert isinstance(lst, list) +# return Array(lst) +# +@as_var("aget") +def aget(self, idx): + assert isinstance(self, Array) + return self._list[idx.int_val()] + +@as_var("aset") +def aset(self, idx, val): + assert isinstance(self, Array) + self._list[idx.int_val()] = val + return val + +@as_var("array-copy") +def array_copy(afrom, afstart, ato, atstart, cnt): + afrom = afrom.array_val() + ato = ato.array_val() + afstart = afstart.int_val() + atstart = atstart.int_val() + for x in range(cnt.int_val()): + ato[atstart + x] = afrom[afstart + x] + + +# +# @as_var("aslice") +# def aslice(self, offset): +# assert isinstance(self, Array) and isinstance(offset, Integer) +# +# offset = offset.int_val() +# if offset >= 0: +# return Array(self._list[offset:]) +# else: +# rt.throw(rt.wrap(u"offset must be an Integer >= 0")) +# +# @as_var("aconcat") +# def aconcat(self, other): +# assert isinstance(self, Array) and isinstance(other, Array) +# return Array(self._list + other._list) +# +# @as_var("alength") +# def alength(self): +# assert isinstance(self, Array) +# return rt.wrap(len(self._list)) +# +# @as_var("make-array") +# def make_array(l): +# affirm(isinstance(l, Integer), u"l must be an Integer") +# return Array([nil] * l.int_val()) +# +# +# # ByteArray +# ARRAY_OF_UCHAR = lltype.Array(lltype.Char) +# +# class ByteArray(object.Object): +# _type = object.Type(u"pixie.stdlib.ByteArray") +# +# def __init__(self, size): +# self._cnt = size +# self._buffer = lltype.malloc(ARRAY_OF_UCHAR, size, flavor="raw") +# for x in range(size): +# self._buffer[x] = chr(0) +# +# def type(self): +# return ByteArray._type +# +# +# def __del__(self): +# lltype.free(self._buffer, flavor="raw") +# +# +# @jit.unroll_safe +# def reduce_small(self, f, init): +# for x in range(self._cnt): +# if rt.reduced_QMARK_(init): +# return rt.deref(init) +# init = f.invoke([init, rt.wrap(ord(self._buffer[x]))]) +# return init +# +# +# def reduce_large(self, f, init): +# for x in range(self._cnt): +# if rt.reduced_QMARK_(init): +# return rt.deref(init) +# init = f.invoke([init, rt.wrap(ord(self._buffer[x]))]) +# return init +# +# +# @as_var("byte-array") +# def _byte_array(size): +# assert isinstance(size, Integer) +# v = size.r_uint_val() +# return ByteArray(v) +# +# @extend(proto._reduce, ByteArray) +# def _reduce(self, f, init): +# assert isinstance(self, ByteArray) +# if self._cnt > UNROLL_IF_SMALLER_THAN: +# return self.reduce_large(f, init) +# return self.reduce_small(f, init) +# +# @extend(proto._nth, ByteArray) +# def _nth(self, idx): +# assert isinstance(self, ByteArray) +# affirm(isinstance(idx, Integer), u"Index must be an integer") +# ival = idx.r_uint_val() +# if 0 <= ival < self._cnt: +# return rt.wrap(ord(self._buffer[ival])) +# +# return affirm(False, u"Index out of Range") +# +# @extend(proto._nth_not_found, ByteArray) +# def _nth_not_found(self, idx, not_found): +# assert isinstance(self, ByteArray) +# affirm(isinstance(idx, Integer), u"Index must be an integer") +# ival = idx.r_uint_val() +# if 0 <= ival < self._cnt: +# return rt.wrap(ord(self._buffer[ival])) +# +# return not_found +# +# @extend(proto._count, ByteArray) +# def _count(self): +# assert isinstance(self, ByteArray) +# return rt.wrap(intmask(self._cnt)) diff --git a/pixie/vm2/arraymap.py b/pixie/vm2/arraymap.py new file mode 100644 index 00000000..7e946d2f --- /dev/null +++ b/pixie/vm2/arraymap.py @@ -0,0 +1,35 @@ +import pixie.vm2.rt as rt +import pixie.vm2.object as object +from pixie.vm2.object import affirm +from pixie.vm2.code import extend, as_var, extend_var +from pixie.vm2.numbers import Integer +from pixie.vm2.primitives import nil +#import pixie.vm.stdlib as proto +import rpython.rlib.jit as jit +from rpython.rtyper.lltypesystem import lltype +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.objectmodel import we_are_translated +from pixie.vm2.keyword import keyword +from pixie.vm2.array import Array +import pixie.vm2.rt as rt +UNROLL_IF_SMALLER_THAN = 8 + +KW_count = keyword(u"count") + + +class ArrayMap(object.Object): + _type = object.Type(u"pixie.stdlib.ArrayMap") + _immutable_fields_ = ["_list"] + def type(self): + return ArrayMap._type + + def __init__(self, lst): + self._list = lst + + def list_val(self): + return self._list + + +@as_var(u"pixie.stdlib", u"array-map-to-array") +def to_list(this): + return Array(this.list_val()) \ No newline at end of file diff --git a/pixie/vm2/bits.py b/pixie/vm2/bits.py new file mode 100644 index 00000000..0d7f63ec --- /dev/null +++ b/pixie/vm2/bits.py @@ -0,0 +1,119 @@ +from pixie.vm2.code import as_var +from pixie.vm2.object import affirm + +from pixie.vm2.numbers import SizeT, Integer +from rpython.rlib.rarithmetic import intmask, r_uint + +import pixie.vm2.rt as rt + +def to_sizet(x): + if isinstance(x, SizeT): + return x + if isinstance(x, Integer): + return SizeT(r_uint(x.r_uint_val())) + + affirm(False, u"Expected something that can be converted to a SizeT") + +@as_var("bit-count32") +def bit_count(i): + i = to_sizet(i).r_uint_val() + i = i - ((i >> 1) & r_uint(0x55555555)) + i = (i & r_uint(0x33333333)) + ((i >> 2) & r_uint(0x33333333)) + return SizeT((((i + (i >> 4) & r_uint(0xF0F0F0F)) * r_uint(0x1010101)) & r_uint(0xffffffff)) >> 24) + + +@as_var("bit-clear") +def bit_clear(x, n): + x = to_sizet(x) + n = to_sizet(n) + return rt.wrap(x.r_uint_val() & ~(r_uint(1) << n.r_uint_val())) + +@as_var("bit-set") +def bit_set(x, n): + x = to_sizet(x) + n = to_sizet(n) + return rt.wrap(x.r_uint_val() | (r_uint(1) << n.r_uint_val())) + +@as_var("bit-flip") +def bit_flip(x, n): + x = to_sizet(x) + n = to_sizet(n) + return rt.wrap(x.r_uint_val() ^ (r_uint(1) << n.r_uint_val())) + +@as_var("bit-not") +def bit_not(x): + x = to_sizet(x) + return rt.wrap(~x.r_uint_val()) + +@as_var("bit-test") +def bit_test(x, n): + x = to_sizet(x) + n = to_sizet(n) + return rt.wrap((x.r_uint_val() & (r_uint(1) << n.r_uint_val())) != r_uint(0)) + +@as_var("bit-and") +def bit_and(x, y): + x = to_sizet(x) + y = to_sizet(y) + return rt.wrap(x.r_uint_val() & y.r_uint_val()) + +@as_var("bit-and-not") +def bit_and_not(x, y): + x = to_sizet(x) + y = to_sizet(y) + return rt.wrap(x.r_uint_val() & ~y.r_uint_val()) + +@as_var("bit-or") +def bit_or(x, y): + x = to_sizet(x) + y = to_sizet(y) + return rt.wrap(x.r_uint_val() | y.r_uint_val()) + +@as_var("bit-xor") +def bit_xor(x, y): + x = to_sizet(x) + y = to_sizet(y) + return rt.wrap(x.r_uint_val() ^ y.r_uint_val()) + +@as_var("bit-shift-left") +def bit_shift_left(x, n): + x = to_sizet(x) + n = to_sizet(n) + return rt.wrap(x.r_uint_val() << n.r_uint_val()) + +@as_var("bit-shift-right") +def bit_shift_right(x, n): + x = to_sizet(x) + n = to_sizet(n) + affirm(isinstance(x, SizeT) and isinstance(n, SizeT), u"x and n must be of type size-t") + return rt.wrap(x.r_uint_val() >> n.r_uint_val()) + +@as_var("unsigned-bit-shift-right") +def unsigned_bit_shift_right(x, n): + x = to_sizet(x) + n = to_sizet(n) + return rt.wrap(intmask(x.r_uint_val() >> n.r_uint_val())) + +digits = "0123456789abcdefghijklmnopqrstuvwxyz" + +@as_var("bit-str") +def bit_str(x, shift): + x = to_sizet(x) + x = x.r_uint_val() + shift = shift.r_uint_val() + + buf = ['_'] * 32 + char_pos = 32 + radix = 1 << shift + mask = radix - 1 + while True: + char_pos -= 1 + buf[char_pos] = digits[x & mask] + x = x >> shift + if x == 0: + break + + res = "" + for i in range(char_pos, char_pos + 32 - char_pos): + res += buf[i] + return rt.wrap(res) diff --git a/pixie/vm2/code.py b/pixie/vm2/code.py new file mode 100644 index 00000000..a6490ebb --- /dev/null +++ b/pixie/vm2/code.py @@ -0,0 +1,1078 @@ +py_object = object +import pixie.vm2.object as object +from pixie.vm2.object import affirm, runtime_error +from pixie.vm2.primitives import nil, false +from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.listsort import TimSort +from rpython.rlib.jit import elidable_promote, promote +from rpython.rlib.objectmodel import we_are_translated +import rpython.rlib.jit as jit +import pixie.vm2.rt as rt + + +BYTECODES = ["LOAD_CONST", + "ADD", + "EQ", + "INVOKE", + "TAIL_CALL", + "DUP_NTH", + "RETURN", + "COND_BR", + "JMP", + "CLOSED_OVER", + "MAKE_CLOSURE", + "SET_VAR", + "POP", + "DEREF_VAR", + "INSTALL", + "LOOP_RECUR", + "ARG", + "PUSH_SELF", + "POP_UP_N", + "MAKE_MULTI_ARITY", + "MAKE_VARIADIC", + "YIELD", + "PUSH_NS"] + +for x in range(len(BYTECODES)): + globals()[BYTECODES[x]] = r_uint(x) + + +@jit.unroll_safe +def resize_list(lst, new_size): + """'Resizes' a list, via reallocation and copy""" + affirm(len(lst) < new_size, u"New list must be larger than old list") + new_list = [None] * new_size + i = r_uint(0) + while i < len(lst): + new_list[i] = lst[i] + i += 1 + return new_list + + +@jit.unroll_safe +def list_copy(from_lst, from_loc, to_list, to_loc, count): + from_loc = r_uint(from_loc) + to_loc = r_uint(to_loc) + count = r_uint(count) + + i = r_uint(0) + while i < count: + to_list[to_loc + i] = from_lst[from_loc + i] + i += 1 + return to_list + + +@jit.unroll_safe +def slice_to_end(from_list, start_pos): + start_pos = r_uint(start_pos) + items_to_copy = len(from_list) - start_pos + new_lst = [None] * items_to_copy + list_copy(from_list, start_pos, new_lst, 0, items_to_copy) + return new_lst + + +@jit.unroll_safe +def slice_from_start(from_list, count, extra=r_uint(0)): + new_lst = [None] * (count + extra) + list_copy(from_list, 0, new_lst, 0, count) + return new_lst + + +# class TailCall(object.Object): +# _type = object.Type("TailCall") +# __immutable_fields_ = ["_f", "_args"] +# def __init__(self, f, args): +# self._f = f +# self._args = args +# +# def run(self): +# return self._f._invoke(self._args) + + +class BaseCode(object.Object): + _immutable_fields_ = ["_meta", "_name"] + def __init__(self): + from pixie.vm2.string import String + assert isinstance(self, BaseCode) + self._name = u"unknown" + self._is_macro = False + self._meta = nil + + def meta(self): + return self._meta + + def with_meta(self, meta): + assert false, "not implemented" + + def name(self): + return self._name + + def set_macro(self): + self._is_macro = True + + def is_macro(self): + assert isinstance(self, BaseCode) + return self._is_macro + + def get_consts(self): + raise NotImplementedError() + + def get_bytecode(self): + raise NotImplementedError() + + @elidable_promote() + def stack_size(self): + return 0 + + def invoke_with(self, args, this_fn): + return self.invoke(args) + +def join_last(words, sep): + """ + Joins by commas and uses 'sep' on last word. + + Eg. join_last(['dog', 'cat', 'rat'] , 'and') = 'dog, cat and rat' + """ + if len(words) == 1: + return words[0] + else: + if len(words) == 2: + s = words[0] + u" " + sep + u" " + words[1] + else: + s = u", ".join(words[0:-1]) + s += u" " + sep + u" " + words[-1] + return s + +class MultiArityFn(BaseCode): + _type = object.Type(u"pixie.stdlib.MultiArityFn") + + _immutable_fields_ = ["_arities[*]", "_required_arity", "_rest_fn", "_name"] + + def type(self): + return MultiArityFn._type + + def __init__(self, name, arities, required_arity=0, rest_fn=None, meta=nil): + BaseCode.__init__(self) + self._name = name + self._arities = arities + self._required_arity = required_arity + self._rest_fn = rest_fn + self._meta = meta + + def with_meta(self, meta): + return MultiArityFn(self._name, self._arities, self._required_arity, self._rest_fn, meta) + + @elidable_promote() + def _get_fn(self, arity): + f = self._arities.get(arity, None) + if f is not None: + return f + if self._rest_fn is not None and arity >= self._required_arity: + return self._rest_fn + + return None + + @jit.unroll_safe + def get_fn(self, arity): + fn = self._get_fn(arity) + if fn: + return fn + + acc = [] + sorted = TimSort(self.get_arities()) + sorted.sort() + for x in sorted.list: + acc.append(unicode(str(x))) + + if self._rest_fn: + acc.append(unicode(str(self._rest_fn.required_arity())) + u"+") + + runtime_error(u"Wrong number of arguments " + unicode(str(arity)) + u" for function '" + unicode(self._name) + u"'. Expected " + join_last(acc, u"or"), + u"pixie.stdlib/InvalidArityException") + + + + + def get_arities(self): + return self._arities.keys() + + def invoke_k(self, args, stack): + return self.invoke_k_with(args, stack, self) + + def invoke_k_with(self, args, stack, self_fn): + return self.get_fn(len(args)).invoke_k_with(args, stack, self_fn) + + +class NativeFn(BaseCode): + """Wrapper for a native function""" + _type = object.Type(u"pixie.stdlib.NativeFn") + + def __init__(self, doc=None): + BaseCode.__init__(self) + + def type(self): + return NativeFn._type + + def invoke(self, args): + return self.inner_invoke(args) + + def invoke_k(self, args, stack): + return self.invoke(args), stack + + def inner_invoke(self, args): + raise NotImplementedError() + + def invoke_with(self, args, this_fn): + return self.invoke(args) + + +class Code(BaseCode): + """Interpreted code block. Contains consts and """ + _type = object.Type(u"pixie.stdlib.Code") + _immutable_fields_ = ["_arity", "_consts[*]", "_bytecode", "_stack_size", "_meta"] + + def type(self): + return Code._type + + def __init__(self, name, arity, bytecode, consts, stack_size, debug_points, meta=nil): + BaseCode.__init__(self) + self._arity = arity + self._bytecode = bytecode + self._consts = consts + self._name = name + self._stack_size = stack_size + self._debug_points = debug_points + self._meta = meta + + def with_meta(self, meta): + return Code(self._name, self._arity, self._bytecode, self._consts, self._stack_size, self._debug_points, meta=meta) + + def get_debug_points(self): + return self._debug_points + + def invoke(self, args): + if len(args) == self.get_arity(): + return self.invoke_with(args, self) + else: + runtime_error(u"Invalid number of arguments " + unicode(str(len(args))) + + u" for function '" + unicode(str(self._name)) + u"'. Expected " + + unicode(str(self.get_arity())), + u":pixie.stdlib/InvalidArityException") + + def invoke_with(self, args, this_fn): + return interpret(self, args, self_obj=this_fn) + + + @elidable_promote() + def get_arity(self): + return self._arity + + @elidable_promote() + def get_consts(self): + return self._consts + + @elidable_promote() + def get_bytecode(self): + return self._bytecode + + @elidable_promote() + def stack_size(self): + return self._stack_size + + @elidable_promote() + def get_base_code(self): + return self + + +class VariadicCode(BaseCode): + _immutable_fields_ = ["_required_arity", "_code", "_meta"] + _type = object.Type(u"pixie.stdlib.VariadicCode") + + def type(self): + return VariadicCode._type + + def __init__(self, code, required_arity, meta=nil): + BaseCode.__init__(self) + self._required_arity = r_uint(required_arity) + self._code = code + self._meta = meta + + def with_meta(self, meta): + return VariadicCode(self._code, self._required_arity, meta) + + def name(self): + return None + + def required_arity(self): + return self._required_arity + + def invoke_k(self, args, stack): + return self.invoke_k_with(args, stack, self) + + def invoke_k_with(self, args, stack, this_fn): + from pixie.vm2.array import Array + argc = len(args) + if self._required_arity == 0: + return self._code.invoke_k([Array(args)], stack) + if argc == self._required_arity: + new_args = resize_list(args, len(args) + 1) + new_args[len(args)] = Array([]) + return self._code.invoke_k_with(new_args, stack, this_fn) + elif argc > self._required_arity: + start = slice_from_start(args, self._required_arity, 1) + rest = slice_to_end(args, self._required_arity) + start[self._required_arity] = Array(rest) + return self._code.invoke_k_with(start, stack, this_fn) + affirm(False, u"Got " + unicode(str(argc)) + u" arg(s) need at least " + unicode(str(self._required_arity))) + return None, stack + + +class Closure(BaseCode): + _type = object.Type(u"pixie.stdlib.Closure") + _immutable_fields_ = ["_closed_overs[*]", "_code", "_meta"] + + def type(self): + return Closure._type + + def __init__(self, code, closed_overs, meta=nil): + BaseCode.__init__(self) + affirm(isinstance(code, Code), u"Code argument to Closure must be an instance of Code") + self._code = code + self._closed_overs = closed_overs + self._meta = meta + + def with_meta(self, meta): + return Closure(self._code, self._closed_overs, meta) + + + def name(self): + return None + + def invoke(self, args): + return self.invoke_with(args, self) + + def invoke_with(self, args, self_fn): + return interpret(self, args, self_obj=self_fn) + + def get_closed_over(self, idx): + return self._closed_overs[idx] + + def get_consts(self): + return self._code.get_consts() + + def get_bytecode(self): + return self._code.get_bytecode() + + def stack_size(self): + return self._code.stack_size() + + def get_closed_overs(self): + return self._closed_overs + + def get_base_code(self): + return self._code.get_base_code() + + def get_debug_points(self): + return self._code.get_debug_points() + + +class Undefined(object.Object): + _type = object.Type(u"pixie.stdlib.Undefined") + + def type(self): + return Undefined._type + +undefined = Undefined() + + +class DynamicVars(py_object): + def __init__(self): + self._vars = [{}] + + def push_binding_frame(self): + self._vars = rt.cons(rt.first(self._vars), self._vars) + + def pop_binding_frame(self): + self._vars = rt.next(self._vars) + + def current_frame(self): + return rt.first(self._vars) + + def get_current_frames(self): + return self._vars + + def set_current_frames(self, vars): + self._vars = vars + + def get_var_value(self, var, not_found): + return rt._val_at(self.current_frame(), var, not_found) + + def set_var_value(self, var, val): + cur_frame = self.current_frame() + self.pop_binding_frame() + self._vars = rt.cons(rt._assoc(cur_frame, var, val), self._vars) + + + + +class Var(BaseCode): + _type = object.Type(u"pixie.stdlib.Var") + _immutable_fields_ = ["_rev?", "_ns", "_name"] + + def type(self): + return Var._type + + def __init__(self, ns, name): + BaseCode.__init__(self) + self._ns = ns + self._name = name + self._rev = 0 + self._root = undefined + self._dynamic = False + + def set_root(self, o): + affirm(o is not None, u"Invalid var set") + self._rev += 1 + self._root = o + return self + + def set_value(self, val): + affirm(self._dynamic, u"Can't set the value of a non-dynamic var") + _dynamic_vars.set_var_value(self, val) + return self + + def set_dynamic(self): + self._dynamic = True + self._rev += 1 + + + def get_dynamic_value(self): + return nil + + + + @elidable_promote() + def _is_dynamic(self, rev): + return self._dynamic + + def is_dynamic(self): + return self._is_dynamic(self._rev) + + @elidable_promote() + def get_root(self, rev): + return self._root + + def deref(self): + if self.is_dynamic(): + if we_are_translated(): + return self.get_dynamic_value() + else: + ## NOT RPYTHON + if globals().has_key("_dynamic_vars"): + return self.get_dynamic_value() + else: + return self.get_root(self._rev) + else: + val = self.get_root(self._rev) + if val is undefined: + pass + affirm(val is not undefined, u"Var " + self._ns + u"/" + self._name + u" is undefined") + return val + + def is_defined(self): + return self._root is not undefined + + def invoke_with(self, args, this_fn): + return self.invoke(args) + + def invoke_k(self, args, stack): + return self.deref().invoke_k(args, stack) + +class bindings(py_object): + def __init__(self, *args): + self._args = list(args) + + def __enter__(self): + _dynamic_vars.push_binding_frame() + for x in range(0, len(self._args), 2): + self._args[x].set_value(self._args[x + 1]) + + def __exit__(self, exc_type, exc_val, exc_tb): + _dynamic_vars.pop_binding_frame() + + +class Refer(py_object): + def __init__(self, ns, refer_syms={}, refer_all=False): + self._namespace = ns + self._refer_syms = refer_syms + self._refer_all = refer_all + self._excludes = {} + + def refer_all(self): + self._refer_all = True + + def add_var_alias(self, old, new): + assert isinstance(old, unicode) + assert isinstance(new, unicode) + + self._refer_syms[new] = old + + def add_var_exclude(self, old): + self._excludes[old] = old + + def find_var(self, name): + if self._refer_all and name not in self._excludes: + return self._namespace.resolve_in_ns(name, False) + else: + other = self._refer_syms.get(name, None) + if other: + return self._namespace.resolve_in_ns(other, False) + return None + + +class Namespace(object.Object): + _type = object.Type(u"pixie.stdlib.Namespace") + + def type(self): + return Namespace._type + + def __init__(self, name): + self._registry = {} + self._name = name + self._refers = {} + self._var_cache = {} + self._rev = 0 + + def reset_cache(self): + self._var_cache.clear() + self._rev += 1 + + def intern_or_make(self, name): + assert name is not None + self.reset_cache() + affirm(isinstance(name, unicode), u"Var names must be unicode") + v = self._registry.get(name, None) + if v is None: + v = Var(self._name, name) + self._registry[name] = v + return v + + def add_refer(self, ns, as_nm=None, refer_all=False): + assert isinstance(ns, Namespace) + self.reset_cache() + + if as_nm is not None: + assert isinstance(as_nm, unicode) + + if as_nm is None: + as_nm = ns._name + + self._refers[as_nm] = Refer(ns, refer_all=refer_all) + + def get_refer(self, nm): + assert isinstance(nm, unicode) + + return self._refers.get(nm, None) + + def add_refer_symbol(self, sym, var): + assert isinstance(self, Namespace) + self.reset_cache() + + name = rt.name(sym) + prev_binding = self._registry.get(name, None) + if prev_binding is not None: + print rt.name(rt.str(rt.wrap(u"Warning: "), sym, rt.wrap(u" already refers to "), prev_binding)) + + self._registry[name] = var + return var + + def include_stdlib(self): + stdlib = ns_registry.find_or_make(u"pixie.stdlib") + self.add_refer(stdlib, refer_all=True) + self.reset_cache() + + + def resolve(self, s, use_refers=True): + import pixie.vm2.symbol as symbol + affirm(isinstance(s, symbol.Symbol), u"Must resolve symbols") + ns = s.get_ns() + name = s.get_name() + + if ns is not None: + refer = self._refers.get(ns, None) + resolved_ns = None + if refer is not None: + resolved_ns = refer._namespace + if resolved_ns is None: + resolved_ns = ns_registry.get(ns, None) + if resolved_ns is None: + affirm(False, u"Unable to resolve namespace: " + ns + u" inside namespace " + self._name) + else: + resolved_ns = self + + assert isinstance(resolved_ns, Namespace) + + return resolved_ns.resolve_in_ns(name, use_refers=use_refers) + + def resolve_in_ns_ex(self, ns, name, use_refers=True): + return self._resolve_in_ns_ex(ns, name, self._rev, use_refers) + + @jit.elidable_promote() + def _resolve_in_ns_ex(self, ns, name, rev, use_refers): + if ns is None: + return self._resolve_in_ns(name, use_refers, rev) + else: + refer = self._refers.get(ns, None) + if refer is not None: + return refer._namespace._resolve_in_ns(name, use_refers, rev) + else: + resolved_ns = ns_registry.get(ns, None) + if resolved_ns is None: + return None + return resolved_ns.resolve_in_ns(name, False) + + def resolve_in_ns(self, name, use_refers=True): + return self._resolve_in_ns(name, use_refers, self._rev) + + + @jit.elidable_promote() + def _resolve_in_ns(self, name, use_refers, rev): + var = self._var_cache.get(name, None) + if var is None: + var = self._registry.get(name, None) + if var is None and use_refers: + for refer_nm in self._refers: + refer = self._refers[refer_nm] + var = refer.find_var(name) + if var is not None: + break + + if var is not None: + self._var_cache[name] = var + + return var + + def get(self, name, default): + return self._registry.get(name, default) + + +class NamespaceRegistry(py_object): + def __init__(self): + self._registry = {} + self._rev = 0 + + @jit.elidable_promote() + def _find_ns(self, name, rev): + #affirm(isinstance(name, unicode), u"Namespace names must be unicode") + v = self._registry.get(name, None) + return v + + + def find_or_make(self, name): + v = self._find_ns(name, self._rev) + if v is None: + self._rev += 1 + v = Namespace(name) + self._registry[name] = v + v.include_stdlib() + return v + + def get(self, name, default): + return self._registry.get(name, default) + +ns_registry = NamespaceRegistry() + + +def intern_var(ns, name=None): + if name is None: + name = ns + ns = u"" + + return ns_registry.find_or_make(ns).intern_or_make(name) + + +def get_var_if_defined(ns, name, els=None): + w_ns = ns_registry.get(ns, None) + if w_ns is None: + return els + return w_ns.get(name, els) + + +class DefaultProtocolFn(NativeFn): + def __init__(self, pfn): + BaseCode.__init__(self) + self._pfn = pfn + + def invoke(self, args): + tp = args[0].type() + assert isinstance(tp, object.Type) + pfn = self._pfn + if isinstance(pfn, PolymorphicFn): + protocol = pfn._protocol + elif isinstance(pfn, DoublePolymorphicFn): + protocol = pfn._protocol + else: + assert False + assert isinstance(protocol, Protocol) + affirm(False, u"No override for " + tp._name + u" on " + self._pfn._name + u" in protocol " + protocol._name) + + +class Protocol(object.Object): + _type = object.Type(u"pixie.stdlib.Protocol") + + _immutable_fields_ = ["_rev?"] + + def type(self): + return Protocol._type + + def __init__(self, name): + self._name = name + self._polyfns = {} + self._satisfies = {} + self._rev = 0 + + def add_method(self, pfn): + self._polyfns[pfn] = pfn + + def add_satisfies(self, tp): + self._satisfies[tp] = tp + self._rev += 1 + + @elidable_promote() + def _get_satisfies(self, tp, rev): + return tp in self._satisfies + + def satisfies(self, tp): + return self._get_satisfies(tp, self._rev) + + +class PolymorphicFn(BaseCode): + _type = object.Type(u"pixie.stdlib.PolymorphicFn") + + def type(self): + return PolymorphicFn._type + + _immutable_fields_ = ["_rev?"] + + def __init__(self, name, protocol): + BaseCode.__init__(self) + self._name = name + self._dict = {} + # stored separately to allow ordered extending (e.g. more general protocols later) + self._protos = [] + self._rev = 0 + self._protocol = protocol + self._default_fn = DefaultProtocolFn(self) + self._fn_cache = {} + protocol.add_method(self) + + def extend(self, tp, fn): + self._dict[tp] = fn + if isinstance(tp, Protocol): + self._protos.append(tp) + self._rev += 1 + self._fn_cache = {} + self._protocol.add_satisfies(tp) + + def satisfied_by(self, o): + if not isinstance(o, object.Type): + o = o.type() + + return self._protocol.satisfies(o) + + + def _find_parent_fn(self, tp): + ## Search the entire object tree to find the function to execute + assert isinstance(tp, object.Type) + + find_tp = tp + while True: + result = self._dict.get(find_tp, None) + if result is not None: + return result + + for proto in self._protos: + if proto.satisfies(find_tp): + return self._dict[proto] + + find_tp = find_tp._parent + if find_tp is None: + break + + return self._default_fn + + def set_default_fn(self, fn): + self._default_fn = fn + self._rev += 1 + self._fn_cache = {} + + @elidable_promote() + def get_protocol_fn(self, tp, rev): + fn = self._fn_cache.get(tp, None) + if fn is None: + fn = self._find_parent_fn(tp) + self._fn_cache[tp] = fn + + return promote(fn) + + def invoke_k(self, args, stack): + affirm(len(args) >= 1, u"Wrong number of args") + a = args[0].type() + fn = self.get_protocol_fn(a, self._rev) + return fn.invoke_k(args, stack) + + +class DoublePolymorphicFn(BaseCode): + """A function that is polymorphic on the first two arguments""" + _type = object.Type(u"pixie.stdlib.DoublePolymorphicFn") + + def type(self): + return DefaultProtocolFn._type + + _immutable_fields_ = ["_rev?"] + + def __init__(self, name, protocol): + BaseCode.__init__(self) + self._name = name + self._dict = {} + self._rev = 0 + self._protocol = protocol + self._default_fn = DefaultProtocolFn(self) + protocol.add_method(self) + + def extend2(self, tp1, tp2, fn): + d1 = self._dict.get(tp1, None) + if d1 is None: + d1 = {} + self._dict[tp1] = d1 + d1[tp2] = fn + self._rev += 1 + self._protocol.add_satisfies(tp1) + + def set_default_fn(self, fn): + self._default_fn = fn + self._rev += 1 + + @elidable_promote() + def get_fn(self, tp1, tp2, _rev): + d1 = self._dict.get(tp1, None) + if d1 is None: + return self._default_fn + fn = d1.get(tp2, self._default_fn) + return promote(fn) + + def invoke_k(self, args, stack): + affirm(len(args) >= 2, u"DoublePolymorphicFunctions take at least two args") + a = args[0].type() + b = args[1].type() + fn = self.get_fn(a, b, self._rev) + return fn.invoke_k(args, stack) + +def munge(s): + return s.replace("-", "_").replace("?", "_QMARK_").replace("!", "_BANG_") + + +import inspect + + +def defprotocol(ns, name, methods): + """Define a protocol in the given namespace with the given name and methods, vars will + be created in the namespace for the protocol and methods. This function will dump + variables for the created protocols/methods in the globals() where this function is called.""" + ns = unicode(ns) + name = unicode(name) + methods = map(unicode, methods) + gbls = inspect.currentframe().f_back.f_globals + proto = Protocol(name) + intern_var(ns, name).set_root(proto) + gbls[munge(name)] = proto + for method in methods: + poly = PolymorphicFn(method, proto) + intern_var(ns, method).set_root(poly) + gbls[munge(method)] = poly + + +def assert_type(x, tp): + affirm(isinstance(x, tp), u"Fatal Error, this should never happen") + return x + + +## PYTHON FLAGS +CO_VARARGS = 0x4 + + +def wrap_fn(fn, tp=object.Object): + """Converts a native Python function into a pixie function.""" + import pixie.vm2.rt as rt + + docstring = unicode(fn.__doc__) if fn.__doc__ else u"" + def as_native_fn(f): + return type("W" + fn.__name__, (NativeFn,), {"inner_invoke": f, "_doc": docstring})() + + def as_variadic_fn(f): + return type("W" + fn.__name__[:len("__args")], (NativeFn,), {"inner_invoke": f, "_doc": docstring})() + + code = fn.func_code + if fn.__name__.endswith("__args"): + return as_variadic_fn(lambda self, args: rt.wrap(fn(args))) + + fn_name = unicode(getattr(fn, "__real_name__", fn.__name__)) + + if code.co_flags & CO_VARARGS: + raise Exception("Variadic functions not supported by wrap") + else: + argc = code.co_argcount + if argc == 0: + def wrapped_fn(self, args): + affirm(len(args) == 0, u"Expected 0 arguments to " + fn_name) + try: + return rt.wrap(fn()) + except object.WrappedException as ex: + #ex._ex._trace.append(object.NativeCodeInfo(fn_name)) + raise + return as_native_fn(wrapped_fn) + + if argc == 1: + def wrapped_fn(self, args): + affirm(len(args) == 1, u"Expected 1 arguments to " + fn_name) + try: + return rt.wrap(fn(args[0])) + except object.WrappedException as ex: + #ex._ex._trace.append(object.NativeCodeInfo(fn_name)) + raise + return as_native_fn(wrapped_fn) + + if argc == 2: + def wrapped_fn(self, args): + affirm(len(args) == 2, u"Expected 2 arguments to " + fn_name) + try: + return rt.wrap(fn(args[0], args[1])) + except object.WrappedException as ex: + #ex._ex._trace.append(object.NativeCodeInfo(fn_name)) + raise + return as_native_fn(wrapped_fn) + if argc == 3: + def wrapped_fn(self, args): + affirm(len(args) == 3, u"Expected 3 arguments to " + fn_name) + + try: + return rt.wrap(fn(args[0], args[1], args[2])) + except object.WrappedException as ex: + #ex._ex._trace.append(object.NativeCodeInfo(fn_name)) + raise + return as_native_fn(wrapped_fn) + + if argc == 4: + def wrapped_fn(self, args): + affirm(len(args) == 4, u"Expected 4 arguments to " + fn_name) + + try: + return rt.wrap(fn(args[0], args[1], args[2], args[3])) + except object.WrappedException as ex: + #ex._ex._trace.append(object.NativeCodeInfo(fn_name)) + raise + return as_native_fn(wrapped_fn) + + if argc == 5: + def wrapped_fn(self, args): + affirm(len(args) == 5, u"Expected 5 arguments to " + fn_name) + + try: + return rt.wrap(fn(args[0], args[1], args[2], args[3], args[4])) + except object.WrappedException as ex: + #ex._ex._trace.append(object.NativeCodeInfo(fn_name)) + raise + return as_native_fn(wrapped_fn) + + assert False, "implement more" + + +def extend(pfn, tp1, tp2=None): + """Extends a protocol to the given Type (not python type), with the decorated function + wraps the decorated function""" + if isinstance(tp1, type): + assert_tp = tp1 + tp1 = tp1._type + else: + assert_tp = object.Object + + def extend_inner(fn): + if tp2 is None: + pfn.extend(tp1, wrap_fn(fn, assert_tp)) + else: + pfn.extend2(tp1, tp2, wrap_fn(fn, assert_tp)) + + return pfn + + return extend_inner + +init_ctx = [] + +def extend_var(ns_name, var_name, tp1): + if isinstance(tp1, type): + assert_tp = tp1 + tp1 = tp1._type + else: + assert_tp = object.Object + + var = intern_var(unicode(ns_name), unicode(var_name)) + + def extend_inner(fn): + init_ctx.append((var, tp1, wrap_fn(fn, assert_tp))) + + return var + + return extend_inner + + + + + +def as_var(ns, name=None): + """Locates a var with the given name (defaulting to the namespace pixie.stdlib), sets + the root to the decorated function. If the function is not an instance of BaseCode it will + be wrapped. """ + if name is None: + name = ns + ns = "pixie.stdlib" + + name = name if isinstance(name, unicode) else unicode(name) + ns = ns if isinstance(ns, unicode) else unicode(ns) + + var = intern_var(ns, name) + + def with_fn(fn): + fn.__real_name__ = name + if not isinstance(fn, object.Object): + fn = wrap_fn(fn) + var.set_root(fn) + return fn + return with_fn + + +def returns(type): + """Tags a var as for unwrapping in rt. When rt imports this var it will be automatically converted to this type""" + def with_fn(fn): + fn._returns = type + return fn + return with_fn + + + +class bindings(py_object): + def __init__(self, *args): + self._args = list(args) + + def __enter__(self): + _dynamic_vars.push_binding_frame() + for x in range(0, len(self._args), 2): + self._args[x].set_value(self._args[x + 1]) + + def __exit__(self, exc_type, exc_val, exc_tb): + _dynamic_vars.pop_binding_frame() + + +def init(): + pass + #globals()["_dynamic_vars"] = DynamicVars() diff --git a/pixie/vm2/custom_types.py b/pixie/vm2/custom_types.py new file mode 100644 index 00000000..abbffc20 --- /dev/null +++ b/pixie/vm2/custom_types.py @@ -0,0 +1,296 @@ +from pixie.vm2.object import Object, Type, affirm, runtime_error +import rpython.rlib.jit as jit +from pixie.vm2.primitives import nil +from rpython.rlib.rarithmetic import r_uint +from pixie.vm2.code import as_var +from pixie.vm2.numbers import Integer, Float +from pixie.vm2.keyword import Keyword, keyword +from pixie.vm2.array import Array +import pixie.vm2.rt as rt + +MAX_FIELDS = 32 + + +def make_ctor_fn(type_name, field_names): + """Constructs a function that will contsruct a Custom type with the given type name + and field names. The resulting function should be called with a list of Objects.""" + + slots = {} + for idx, val in enumerate(field_names): + slots[keyword(val)] = idx + + type = CustomType(type_name, slots) + + def ctor(itms): + return new_inst(type, itms) + + return ctor + + +class CustomType(Type): + _immutable_fields_ = ["_slots", "_rev?"] + def __init__(self, name, slots): + Type.__init__(self, name) + + self._slots = slots + self._mutable_slots = {} + self._rev = 0 + + @jit.elidable_promote() + def get_slot_idx(self, nm): + return self._slots.get(nm, -1) + + def set_mutable(self, nm): + if not self.is_mutable(nm): + self._rev += 1 + self._mutable_slots[nm] = nm + + + @jit.elidable_promote() + def _is_mutable(self, nm, rev): + return nm in self._mutable_slots + + def is_mutable(self, nm): + return self._is_mutable(nm, self._rev) + + @jit.elidable_promote() + def get_num_slots(self): + return len(self._slots) + + def custom_type_name(self): + return self._name + +class CustomTypeInstance(Object): + _immutable_fields_ = ["_type"] + def __init__(self, type): + affirm(isinstance(type, CustomType), u"Can't create a instance of a non custom type") + self._custom_type = type + + def type(self): + return self._custom_type + + def set_field(self, name, val): + idx = self._custom_type.get_slot_idx(name) + assert isinstance(name, Keyword) + if idx == -1: + runtime_error(u"Invalid field named " + name._str + u" on type " + self._custom_type.custom_type_name(), + u"pixie.stdlib/InvalidFieldException") + + old_val = self.get_field_by_idx(idx) + if isinstance(old_val, AbstractMutableCell): + old_val.set_mutable_cell_value(self._custom_type, self, name, idx, val) + else: + self._custom_type.set_mutable(name) + self.set_field_by_idx(idx, val) + return self + + @jit.elidable_promote() + def _get_field_immutable(self, idx, rev): + return self.get_field_by_idx(idx) + + def get_field_immutable(self, idx): + tp = self._custom_type + assert isinstance(tp, CustomType) + return self._get_field_immutable(idx, tp._rev) + + def get_field(self, name, not_found): + idx = self._custom_type.get_slot_idx(name) + assert isinstance(name, Keyword) + if idx == -1: + return not_found + + if self._custom_type.is_mutable(name): + value = self.get_field_by_idx(idx) + else: + value = self.get_field_immutable(idx) + + if isinstance(value, AbstractMutableCell): + return value.get_mutable_cell_value() + else: + return value + + +create_type_prefix = """ +def new_inst(tp, fields): + l = len(fields) + if l >= {max_c}: + runtime_error(u"Too many fields for type", u"pixie.stdlib/TooManyFields") +""" + +clause_template = """ + elif l == {c}: + return CustomType{c}(tp, fields) +""" + +def gen_ct_code(): + acc = create_type_prefix.format(max_c=MAX_FIELDS + 1) + for x in range(MAX_FIELDS + 1): + acc += clause_template.format(c=x) + + #print acc + return acc + +exec gen_ct_code() + + +type_template = """ +class CustomType{c}(CustomTypeInstance): + def __init__(self, tp, fields): + CustomTypeInstance.__init__(self, tp) + {self_fields_list} = fields + + def get_field_by_idx(self, idx): + if idx >= {max}: + return None +""" + +get_field_by_idx_template = """ + elif idx == {c}: + return self._custom_field{c} +""" + +set_field_prefix_template = """ + def set_field_by_idx(self, idx, val): + if idx >= {max}: + return +""" + +set_field_by_idx_template = """ + elif idx == {c}: + self._custom_field{c} = val +""" + +get_list_template = """ + def get_list(self): + return {d} + +""" + +def gen_ct_class_code(): + acc = "" + for x in range(MAX_FIELDS + 1): + if x == 0: + self_fields_list = "_null_" + elif x == 1: + self_fields_list = "self._custom_field0," + else: + self_fields_list = ",".join(map(lambda x: "self._custom_field" + str(x), range(x))) + acc += type_template.format(c=x, self_fields_list=self_fields_list, max=x) + for y in range(x): + acc += get_field_by_idx_template.format(c=y) + + acc += set_field_prefix_template.format(max=x) + + for y in range(x): + acc += set_field_by_idx_template.format(c=y) + + acc += get_list_template.format(d="[" + ", ".join(map(lambda x: "self._custom_field" + str(x), range(x))) + "]") + + + #print acc + return acc + +exec gen_ct_class_code() + + +@as_var("create-type") +def create_type(type_name, fields): + affirm(isinstance(type_name, Keyword), u"Type name must be a keyword") + affirm(isinstance(fields, Array), u"Type names must be in an array") + assert isinstance(fields, Array) + acc = {} + + for i, val in enumerate(fields._list): + affirm(isinstance(val, Keyword), u"Field names must be keywords") + acc[val] = i + + from pixie.vm2.string import String + assert isinstance(type_name, Keyword) + return CustomType(type_name._str, acc) + +@as_var("new") +@jit.unroll_safe +def _new__args(args): + affirm(len(args) >= 1, u"new takes at least one parameter") + tp = args[0] + affirm(isinstance(tp, CustomType), u"Can only create a new instance of a custom type") + cnt = len(args) - 1 + affirm(cnt - 1 != tp.get_num_slots(), u"Wrong number of initializer fields to custom type") + arr = [None] * cnt + for x in range(cnt): + val = args[x + 1] + if isinstance(val, Integer): + val = IntegerMutableCell(val.int_val()) + elif isinstance(val, Float): + val = FloatMutableCell(val.float_val()) + + arr[x] = val + return new_inst(tp, arr) + +@as_var("set-field!") +def set_field(inst, field, val): + affirm(isinstance(inst, CustomTypeInstance), u"Can only set fields on CustomType instances") + affirm(isinstance(field, Keyword), u"Field must be a keyword") + inst.set_field(field, val) + return inst + +@as_var("get-field") +def get_field__args(args): + if len(args) == 2: + inst, field = args + not_found = nil + elif len(args) == 3: + inst, field, not_found = args + else: + runtime_error(u"Get Field requires one or two args") + return nil + + affirm(isinstance(inst, CustomTypeInstance), u"Can only get fields on CustomType instances") + affirm(isinstance(field, Keyword), u"Field must be a keyword") + return inst.get_field(field, not_found) + + +class AbstractMutableCell(Object): + _type = Type(u"pixie.stdlib.AbstractMutableCell") + def type(self): + return self._type + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + pass + + def get_mutable_cell_value(self): + pass + +class IntegerMutableCell(AbstractMutableCell): + def __init__(self, int_val): + self._mutable_integer_val = int_val + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + if not isinstance(value, Integer): + ct.set_mutable(nm) + if isinstance(value, Float): + fields.set_field_by_idx(idx, FloatMutableCell(value.float_val())) + else: + fields.set_field_by_idx(idx, value) + else: + self._mutable_integer_val = value.int_val() + + def get_mutable_cell_value(self): + return rt.wrap(self._mutable_integer_val) + +class FloatMutableCell(AbstractMutableCell): + def __init__(self, float_val): + self._mutable_float_val = float_val + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + if not isinstance(value, Float): + ct.set_mutable(nm) + if isinstance(value, Integer): + fields.set_field_by_idx(idx, IntegerMutableCell(value.int_val())) + else: + fields.set_field_by_idx(idx, value) + else: + self._mutable_float_val = value.float_val() + + def get_mutable_cell_value(self): + return rt.wrap(self._mutable_float_val) \ No newline at end of file diff --git a/pixie/vm2/ffi.py b/pixie/vm2/ffi.py new file mode 100644 index 00000000..2d415587 --- /dev/null +++ b/pixie/vm2/ffi.py @@ -0,0 +1,933 @@ +py_object = object +import rpython.rlib.rdynload as dynload +import pixie.vm2.object as object +from pixie.vm2.object import runtime_error +from pixie.vm2.keyword import Keyword +import pixie.vm2.stdlib as proto +from pixie.vm2.code import as_var, affirm, extend_var, wrap_fn, intern_var +import pixie.vm2.rt as rt +from rpython.rtyper.lltypesystem import rffi, lltype, llmemory +from pixie.vm2.primitives import nil, true, false +from pixie.vm2.numbers import Integer, Float +from pixie.vm2.string import String +from pixie.vm2.keyword import Keyword +from rpython.rlib import clibffi +from rpython.rlib.jit_libffi import jit_ffi_call, CIF_DESCRIPTION, CIF_DESCRIPTION_P, \ + FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG +import rpython.rlib.jit_libffi as jit_libffi +from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated +import rpython.rlib.jit as jit +from rpython.rlib.rarithmetic import intmask + + +""" +FFI interface for pixie. + +This code gets a bit interesting. We use the RPython rlib module jit_libffi to do the interfacing, you can find +good docs in that module. + +""" + + +class CType(object.Type): + def __init__(self, name): + object.Type.__init__(self, name) + + +load_paths = intern_var(u"pixie.stdlib", u"load-paths") + +class ExternalLib(object.Object): + _type = object.Type(u"pixie.stdlib.ExternalLib") + + def type(self): + return ExternalLib._type + + def __init__(self, nm): + assert isinstance(nm, unicode) + self._name = nm + self._is_inited = False + self.load_lib() + + def load_lib(self): + if not self._is_inited: + s = rffi.str2charp(str(self._name)) + try: + self._dyn_lib = dynload.dlopen(s) + self._is_inited = True + finally: + rffi.free_charp(s) + + + def get_fn_ptr(self, nm): + assert isinstance(nm, unicode) + s = rffi.str2charp(str(nm)) + sym = dynload.dlsym(self._dyn_lib, s) + rffi.free_charp(s) + return sym + +@as_var("-ffi-library") +def _ffi_library(ns): + try: + return ExternalLib(ns.get_name()) + except dynload.DLOpenError as ex: + return nil + +class FFIFn(object.Object): + _type = object.Type(u"pixie.stdlib.FFIFn") + _immutable_fields_ = ["_is_inited", "_lib", "_name", "_arg_types[*]", "_arity", "_ret_type", "_is_variadic", \ + "_transfer_size", "_arg0_offset", "_ret_offset", "_cd"] + + def type(self): + return FFIFn._type + + def __init__(self, lib, name, arg_types, ret_type, is_variadic): + self._rev = 0 + self._name = name + self._lib = lib + self._arg_types = arg_types + self._arity = len(arg_types) + self._ret_type = ret_type + self._is_variadic = is_variadic + self._f_ptr = self._lib.get_fn_ptr(self._name) + self._cd = CifDescrBuilder(self._arg_types, self._ret_type).rawallocate() + + @jit.unroll_safe + def prep_exb(self, args): + size = jit.promote(self._cd.exchange_size) + exb = rffi.cast(rffi.VOIDP, lltype.malloc(rffi.CCHARP.TO, size, flavor="raw")) + tokens = [None] * len(args) + + for i, tp in enumerate(self._arg_types): + offset_p = rffi.ptradd(exb, jit.promote(self._cd.exchange_args[i])) + tokens[i] = tp.ffi_set_value(offset_p, args[i]) + + return exb, tokens + + def get_ret_val_from_buffer(self, exb): + offset_p = rffi.ptradd(exb, jit.promote(self._cd.exchange_result)) + ret_val = self._ret_type.ffi_get_value(offset_p) + return ret_val + + @jit.unroll_safe + def _invoke(self, args): + arity = len(args) + if self._is_variadic: + if arity < self._arity: + runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + + u", expected at least " + unicode(str(self._arity))) + else: + if arity != self._arity: + runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + + u", expected " + unicode(str(self._arity))) + + exb, tokens = self.prep_exb(args) + cd = jit.promote(self._cd) + #fp = jit.promote(self._f_ptr) + jit_ffi_call(cd, + self._f_ptr, + exb) + ret_val = self.get_ret_val_from_buffer(exb) + + for x in range(len(args)): + t = tokens[x] + if t is not None: + t.finalize_token() + + lltype.free(exb, flavor="raw") + keepalive_until_here(args) + return ret_val + + def invoke_k(self, args, stack): + self = jit.promote(self) + return self._invoke(args), stack + + + +@as_var("ffi-fn") +def _ffi_fn__args(args): + from pixie.vm2.array import Array + + affirm(len(args) >= 4, u"ffi-fn requires at least 4 arguments") + lib, nm, arg_types, ret_type = args[:4] + + affirm(isinstance(lib, ExternalLib), u"First argument must be an ExternalLib") + affirm(isinstance(ret_type, object.Type), u"Ret type must be a type") + affirm(nm.get_ns() is None, u"Name must not be namespaced") + affirm(isinstance(arg_types, Array), u"Arguments must be in an array") + assert isinstance(arg_types, Array) + + new_args = arg_types._list + + kwargs = args[4:] + affirm(len(kwargs) & 0x1 == 0, u"ffi-fn requires even number of options") + + is_variadic = False + for i in range(0, len(kwargs)/2, 2): + key = kwargs[i] + val = kwargs[i+1] + + affirm(isinstance(key, Keyword), u"ffi-fn options should be keyword/bool pairs") + affirm(val is true or val is false, u"ffi-fn options should be keyword/bool pairs") + + k = key.get_name() + if k == u"variadic?": + is_variadic = True if val is true else False + else: + affirm(False, u"unknown ffi-fn option: :" + k) + + f = FFIFn(lib, nm.get_name(), new_args, ret_type, is_variadic) + return f + +@as_var("ffi-voidp") +def _ffi_voidp(lib, nm): + affirm(isinstance(lib, ExternalLib), u"First argument to ffi-voidp should be an external library") + name = nm.get_name() + return VoidP(lib.get_fn_ptr(name)) + + + + + + + +class Buffer(object.Object): + """ Defines a byte buffer with non-gc'd (therefore non-movable) contents + """ + _type = object.Type(u"pixie.stdlib.Buffer") + + def type(self): + return Buffer._type + + def __init__(self, size): + self._size = size + self._used_size = 0 + self._buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor="raw") + + + def __del__(self): + #lltype.free(self._buffer, flavor="raw") + pass + + def set_used_size(self, size): + self._used_size = size + + def buffer(self): + return self._buffer + + def raw_data(self): + return rffi.cast(rffi.VOIDP, self._buffer) + + def count(self): + return self._used_size + + def nth_char(self, idx): + assert isinstance(idx, int) + return self._buffer[idx] + + def capacity(self): + return self._size + + def free_data(self): + lltype.free(self._buffer, flavor="raw") + + +# +# @extend(proto._dispose_BANG_, Buffer) +# def _dispose_voidp(self): +# self.free_data() +# +# +# @extend(proto._nth, Buffer) +# def _nth(self, idx): +# return rt.wrap(ord(self.nth_char(idx.int_val()))) +# +# @extend(proto._nth_not_found, Buffer) +# def _nth_not_found(self, idx, not_found): +# return rt.wrap(ord(self.nth_char(idx.int_val()))) +# +# + +@extend_var("pixie.stdlib", "-count", Buffer) +def _count(self): + return rt.wrap(intmask(self.count())) + +@as_var("buffer") +def buffer(size): + return Buffer(size.int_val()) + +@as_var("buffer-capacity") +def buffer_capacity(buffer): + return rt.wrap(intmask(buffer.capacity())) + +@as_var("set-buffer-count!") +def set_buffer_size(self, size): + self.set_used_size(size.int_val()) + return self + +def make_itype(name, ctype, llt): + lltp = lltype.Ptr(lltype.Array(llt, hints={'nolength': True})) + class GenericCInt(CType): + def __init__(self): + CType.__init__(self, name) + + def ffi_get_value(self, ptr): + casted = rffi.cast(lltp, ptr) + return Integer(rffi.cast(rffi.LONG, casted[0])) + + def ffi_set_value(self, ptr, val): + casted = rffi.cast(lltp, ptr) + casted[0] = rffi.cast(llt, val.int_val()) + + def ffi_size(self): + return rffi.sizeof(llt) + + def ffi_type(self): + return ctype + + return GenericCInt() + +from rpython.rlib.rarithmetic import build_int +for x in [8, 16, 32, 64]: + for s in [True, False]: + nm = "C" + ("" if s else "U") + "Int" + str(x) + int_tp = lltype.build_number(None, build_int(nm, s, x)) + ctype = clibffi.cast_type_to_ffitype(int_tp) + make_itype(unicode("pixie.stdlib." + nm), ctype, int_tp) + + + + + + + +class Token(py_object): + """ Tokens are returned by ffi_set_value and are called when ffi is ready to clean up resources + """ + def finalize_token(self): + pass + + + +class CInt(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CInt") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.INTP, ptr) + return Integer(rffi.cast(rffi.LONG, casted[0])) + + def ffi_set_value(self, ptr, val): + casted = rffi.cast(rffi.INTP, ptr) + casted[0] = rffi.cast(rffi.INT, val.int_val()) + + def ffi_size(self): + return rffi.sizeof(rffi.INT) + + def ffi_type(self): + return clibffi.cast_type_to_ffitype(rffi.INT) +CInt() + +class CDouble(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CDouble") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.DOUBLEP, ptr) + return Float(casted[0]) + + def ffi_set_value(self, ptr, val): + casted = rffi.cast(rffi.DOUBLEP, ptr) + casted[0] = rffi.cast(rffi.DOUBLE, val.float_val()) + + def ffi_size(self): + return rffi.sizeof(rffi.DOUBLE) + + def ffi_type(self): + return clibffi.cast_type_to_ffitype(rffi.DOUBLE) +CDouble() + +class CCharP(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CCharP") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.CCHARPP, ptr) + if casted[0] == lltype.nullptr(rffi.CCHARP.TO): + return nil + else: + return String(unicode(rffi.charp2str(casted[0]))) + + def ffi_set_value(self, ptr, val): + if isinstance(val, String): + pnt = rffi.cast(rffi.CCHARPP, ptr) + utf8 = unicode_to_utf8(val.get_name()) + raw = rffi.str2charp(utf8) + pnt[0] = raw + return CCharPToken(raw) + elif isinstance(val, Buffer): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = val.buffer() + elif isinstance(val, VoidP): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = val.raw_data() + elif val is nil: + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = rffi.cast(rffi.VOIDP, 0) + elif isinstance(val, CStruct): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = rffi.cast(rffi.VOIDP, val.raw_data()) + else: + frm_name = val.to_repr() + to_name = self.to_repr() + affirm(False, u"Cannot encode " + frm_name + u" as " + to_name) + + + def ffi_size(self): + return rffi.sizeof(rffi.CCHARP) + + def ffi_type(self): + return clibffi.ffi_type_pointer +CCharP() + +class CCharPToken(Token): + def __init__(self, raw): + self._raw = raw + + def finalize_token(self): + rffi.free_charp(self._raw) + + + +class CVoid(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CVoid") + + def ffi_get_value(self, ptr): + return nil + + def ffi_set_value(self, ptr, val): + runtime_error(u"Can't encode a Void") + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer + +cvoid = CVoid() + +class CVoidP(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CVoidP") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.VOIDPP, ptr) + if casted[0] == lltype.nullptr(rffi.VOIDP.TO): + return nil + else: + return VoidP(casted[0]) + + def ffi_set_value(self, ptr, val): + pnt = rffi.cast(rffi.VOIDPP, ptr) + if isinstance(val, String): + pnt = rffi.cast(rffi.CCHARPP, ptr) + utf8 = unicode_to_utf8(val.get_name()) + raw = rffi.str2charp(utf8) + pnt[0] = raw + return CCharPToken(raw) + elif isinstance(val, Buffer): + pnt[0] = val.buffer() + elif isinstance(val, VoidP): + pnt[0] = val.raw_data() + elif val is nil: + pnt[0] = rffi.cast(rffi.VOIDP, 0) + elif isinstance(val, CStruct): + pnt[0] = rffi.cast(rffi.VOIDP, val.raw_data()) + else: + frm_name = val.to_repr() + to_name = self.to_repr() + affirm(False, u"Cannot encode " + frm_name + u" as " + to_name) + + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer +cvoidp = CVoidP() + +class VoidP(object.Object): + def type(self): + return cvoidp + + def __init__(self, raw_data): + self._raw_data = raw_data + + def raw_data(self): + return rffi.cast(rffi.VOIDP, self._raw_data) + + def free_data(self): + lltype.free(self._raw_data, flavor="raw") + +# @extend(proto._dispose_BANG_, cvoidp) +# def _dispose_voidp(self): +# self.free_data() + + + +@as_var(u"pixie.ffi", u"prep-string") +def prep_string(s): + """Takes a Pixie string and returns a VoidP to that string. The string should be freed via dispose!, otherwise + memory leaks could result.""" + affirm(isinstance(s, String), u"Can only prep strings with prep-string") + utf8 = unicode_to_utf8(s.get_name()) + raw = rffi.str2charp(utf8) + return VoidP(rffi.cast(rffi.VOIDP, raw)) + +@as_var(u"pixie.ffi", u"unpack") +def unpack(ptr, offset, tp): + """(unpack ptr offset tp) + Reads a value of type tp from offset of ptr.""" + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + affirm(isinstance(tp, CType), u"Packing type must be a CType") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + return tp.ffi_get_value(ptr) + +@as_var(u"pixie.ffi", u"pack!") +def pack(ptr, offset, tp, val): + """(pack! ptr offset tp val) + Writes val at offset of ptr with the format tp""" + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + affirm(isinstance(tp, CType), u"Packing type must be a CType") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + tp.ffi_set_value(ptr, val) + return nil + +@as_var(u"pixie.ffi", u"ptr-add") +def pack(ptr, offset): + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + return VoidP(ptr) + +class CStructType(object.Type): + base_type = object.Type(u"pixie.ffi.CStruct") + _immutable_fields_ = ["_desc", "_size"] + + def __init__(self, name, size, desc): + object.Type.__init__(self, name, CStructType.base_type) + self._desc = desc + self._size = size + #offsets is a dict of {nm, (type, offset)} + + def get_offset(self, nm): + (tp, offset) = self._desc.get(nm, (None, 0)) + + assert tp is not None + + return offset + + def get_type(self, nm): + (tp, offset) = self._desc.get(nm, (None, 0)) + + assert tp is not None + + return tp + + def get_size(self): + return self._size + + def cast_to(self, frm): + return CStruct(self, frm.raw_data()) + + @jit.elidable_promote() + def get_desc(self, nm): + return self._desc.get(nm, (None, 0)) + + def invoke(self, args): + return CStruct(self, rffi.cast(rffi.VOIDP, lltype.malloc(rffi.CCHARP.TO, self._size, flavor="raw"))) + + +registered_callbacks = {} + +class CCallback(object.Object): + _type = object.Type(u"pixie.ffi.CCallback") + + def __init__(self, cft, raw_closure, id, fn): + self._fn = fn + self._cft = cft + self._raw_closure = raw_closure + self._is_invoked = False + self._unique_id = id + + def type(self): + return CCallback._type + + def get_raw_closure(self): + return self._raw_closure + + def ll_invoke(self, llargs, llres): + cft = self._cft + assert isinstance(cft, CFunctionType) + + args = [None] * len(cft._arg_types) + for i, tp in enumerate(cft._arg_types): + args[i] = tp.ffi_get_value(llargs[i]) + + self._is_invoked = True + retval = self._fn.invoke(args) + if cft._ret_type is not cvoid: + cft._ret_type.ffi_set_value(llres, retval) + + + def cleanup(self): + del registered_callbacks[self._unique_id] + clibffi.closureHeap.free(self._raw_closure) + +# @extend(proto._dispose_BANG_, CCallback) +# def _dispose(self): +# self.cleanup() + + + +@as_var(u"pixie.ffi", u"-ffi-callback") +def ffi_callback(args, ret_type): + """(ffi-callback args ret-type) + Creates a ffi callback type. Args is a vector of CType args. Ret-type is the CType return + type of the callback. Returns a ffi callback type that can be used with ffi-prep-callback.""" + from pixie.vm2.array import Array + assert isinstance(args, Array) + args_w = args._list + + return CFunctionType(args_w, ret_type) + +@as_var(u"pixie.ffi", u"ffi-prep-callback") +def ffi_prep_callback(tp, f): + """(ffi-prep-callback callback-tp fn) + Prepares a Pixie function for use as a c callback. callback-tp is a ffi callback type, + fn is a pixie function (can be a closure, native fn or any object that implements -invoke. + Returns a function pointer that can be passed to c and invoked as a callback.""" + affirm(isinstance(tp, CFunctionType), u"First argument to ffi-prep-callback must be a CFunctionType") + raw_closure = rffi.cast(rffi.VOIDP, clibffi.closureHeap.alloc()) + + unique_id = rffi.cast(lltype.Signed, raw_closure) + + res = clibffi.c_ffi_prep_closure(rffi.cast(clibffi.FFI_CLOSUREP, raw_closure), tp.get_cd().cif, + invoke_callback, + rffi.cast(rffi.VOIDP, unique_id)) + + + if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: + registered_callbacks[unique_id] = None + runtime_error(u"libffi failed to build this callback") + + cb = CCallback(tp, raw_closure, unique_id, f) + registered_callbacks[unique_id] = cb + + return cb + +# Callback Code + +@jit.jit_callback("CFFI") +def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): + cb = registered_callbacks[rffi.cast(rffi.INT_real, ll_userdata)] + cb.ll_invoke(ll_args, ll_res) + +class FunctionTypeNameGenerator(py_object): + def __init__(self): + self._idx = 0 + def next(self): + self._idx += 1 + return unicode("CFunctionType" + str(self._idx)) + +name_gen = FunctionTypeNameGenerator() + +class CFunctionType(object.Type): + base_type = object.Type(u"pixie.ffi.CType") + _immutable_fields_ = ["_arg_types", "_ret-type", "_cd"] + + def __init__(self, arg_types, ret_type): + object.Type.__init__(self, name_gen.next(), CStructType.base_type) + self._arg_types = arg_types + self._ret_type = ret_type + self._cd = CifDescrBuilder(self._arg_types, self._ret_type).rawallocate() + + def ffi_get_value(self, ptr): + runtime_error(u"Cannot get a callback value via FFI") + + def ffi_set_value(self, ptr, val): + affirm(isinstance(val, CCallback), u"Can only encode CCallbacks as function pointers") + + + casted = rffi.cast(rffi.VOIDPP, ptr) + casted[0] = val.get_raw_closure() + + return None + + def get_cd(self): + return self._cd + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer + +class CStruct(object.Object): + _immutable_fields_ = ["_type", "_buffer"] + def __init__(self, tp, buffer): + self._type = tp + self._buffer = buffer + + def type(self): + return self._type + + def raw_data(self): + return rffi.cast(rffi.VOIDP, self._buffer) + + def val_at(self, k, not_found): + (tp, offset) = self._type.get_desc(k) + + if tp is None: + return not_found + + offset = rffi.ptradd(self._buffer, offset) + return tp.ffi_get_value(rffi.cast(rffi.VOIDP, offset)) + + def set_val(self, k, v): + (tp, offset) = self._type.get_desc(k) + + if tp is None: + runtime_error(u"Invalid field name: " + k.to_repr()) + + offset = rffi.ptradd(self._buffer, offset) + tp.ffi_set_value(rffi.cast(rffi.VOIDP, offset), v) + + return nil + + def free_data(self): + lltype.free(self._buffer, flavor="raw") + +@wrap_fn +def _dispose_cstruct(self): + self.free_data() + + + + +@as_var("pixie.ffi", "c-struct") +def c_struct(name, size, spec): + """(c-struct name size spec) + Creates a CStruct named name, of size size, with the given spec. Spec is a vector + of vectors. Each row of the format [field-name type offset]""" + from pixie.vm2.array import Array + from pixie.vm2.code import intern_var + d = {} + assert isinstance(spec, Array) + spec_lst = spec._list + for x in range(len(spec_lst)): + row = spec_lst[x] + assert isinstance(row, Array) + row_lst = row._list + nm = row_lst[0] + tp = row_lst[1] + offset = row_lst[2] + + affirm(isinstance(nm, Keyword), u"c-struct field names must be keywords") + if not isinstance(tp, CType): + runtime_error(u"c-struct field types must be c types, got: " + tp.to_repr()) + + d[nm] = (tp, offset.int_val()) + + tp = CStructType(name.get_name(), size.int_val(), d) + + dispose_var = intern_var(u"pixie.stdlib", u"-dispose!") + dispose_var.deref().extend(tp, _dispose_cstruct) + return tp + +@as_var("pixie.ffi", "cast") +def c_cast(frm, to): + """(cast from to) + Converts a VoidP to a CStruct. From is either a VoidP or a CStruct, to is a CStruct type.""" + if not isinstance(to, CStructType): + runtime_error(u"Expected a CStruct type to cast to, got " + to.to_repr()) + + if not isinstance(frm, CStruct) and not isinstance(frm, VoidP): + runtime_error(u"From must be a CVoidP or a CStruct, got " + to.to_repr()) + + return to.cast_to(frm) + +@as_var("pixie.ffi", "struct-size") +def struct_size(tp): + """(struct-size tp) + Gives the size of the given CStruct type tp, in bytes.""" + if not isinstance(tp, CStructType): + runtime_error(u"Expected a CStruct type to get the size of, got " + tp.to_repr()) + + return rt.wrap(tp.get_size()) + +# @extend(proto._val_at, CStructType.base_type) +# def val_at(self, k, not_found): +# return self.val_at(k, not_found) + +@as_var("pixie.ffi", "set!") +def set_(self, k, val): + """(set! ptr k val) + Sets a field k of struct ptr to value val""" + return self.set_val(k, val) + + + +@as_var("pixie.ffi", "prep-ffi-call") +def prep_ffi_call__args(args): + fn = args[0] + affirm(isinstance(fn, CFunctionType), u"First arg must be a FFI function") + + + + + +import sys + +USE_C_LIBFFI_MSVC = getattr(clibffi, 'USE_C_LIBFFI_MSVC', False) + + + +class CifDescrBuilder(py_object): + rawmem = lltype.nullptr(rffi.CCHARP.TO) + + def __init__(self, fargs, fresult): + self.fargs = fargs + self.fresult = fresult + + def fb_alloc(self, size): + size = llmemory.raw_malloc_usage(size) + if not self.bufferp: + self.nb_bytes += size + return lltype.nullptr(rffi.CCHARP.TO) + else: + result = self.bufferp + self.bufferp = rffi.ptradd(result, size) + return result + + def fb_fill_type(self, ctype, is_result_type): + return ctype.ffi_type() + + + def fb_build(self): + # Build a CIF_DESCRIPTION. Actually this computes the size and + # allocates a larger amount of data. It starts with a + # CIF_DESCRIPTION and continues with data needed for the CIF: + # + # - the argument types, as an array of 'ffi_type *'. + # + # - optionally, the result's and the arguments' ffi type data + # (this is used only for 'struct' ffi types; in other cases the + # 'ffi_type *' just points to static data like 'ffi_type_sint32'). + # + nargs = len(self.fargs) + + # start with a cif_description (cif and exchange_* fields) + self.fb_alloc(llmemory.sizeof(CIF_DESCRIPTION, nargs)) + + # next comes an array of 'ffi_type*', one per argument + atypes = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * nargs) + self.atypes = rffi.cast(FFI_TYPE_PP, atypes) + + # next comes the result type data + self.rtype = self.fb_fill_type(self.fresult, True) + + # next comes each argument's type data + for i, farg in enumerate(self.fargs): + atype = self.fb_fill_type(farg, False) + if self.atypes: + self.atypes[i] = atype + + def align_arg(self, n): + return (n + 7) & ~7 + + def fb_build_exchange(self, cif_descr): + nargs = len(self.fargs) + + # first, enough room for an array of 'nargs' pointers + exchange_offset = rffi.sizeof(rffi.VOIDP) * nargs + exchange_offset = self.align_arg(exchange_offset) + cif_descr.exchange_result = exchange_offset + + # then enough room for the result, rounded up to sizeof(ffi_arg) + exchange_offset += max(rffi.getintfield(self.rtype, 'c_size'), + SIZE_OF_FFI_ARG) + + # loop over args + for i, farg in enumerate(self.fargs): + #if isinstance(farg, W_CTypePointer): + # exchange_offset += 1 # for the "must free" flag + exchange_offset = self.align_arg(exchange_offset) + cif_descr.exchange_args[i] = exchange_offset + exchange_offset += rffi.getintfield(self.atypes[i], 'c_size') + + # store the exchange data size + cif_descr.exchange_size = exchange_offset + + def fb_extra_fields(self, cif_descr): + cif_descr.abi = clibffi.FFI_DEFAULT_ABI # XXX + cif_descr.nargs = len(self.fargs) + cif_descr.rtype = self.rtype + cif_descr.atypes = self.atypes + + @jit.dont_look_inside + def rawallocate(self): + + # compute the total size needed in the CIF_DESCRIPTION buffer + self.nb_bytes = 0 + self.bufferp = lltype.nullptr(rffi.CCHARP.TO) + self.fb_build() + + # allocate the buffer + if we_are_translated(): + rawmem = lltype.malloc(rffi.CCHARP.TO, self.nb_bytes, + flavor='raw') + rawmem = rffi.cast(CIF_DESCRIPTION_P, rawmem) + else: + # gross overestimation of the length below, but too bad + rawmem = lltype.malloc(CIF_DESCRIPTION_P.TO, self.nb_bytes, + flavor='raw') + + # the buffer is automatically managed from the W_CTypeFunc instance + # ctypefunc._cd = rawmem + + # call again fb_build() to really build the libffi data structures + self.bufferp = rffi.cast(rffi.CCHARP, rawmem) + self.fb_build() + assert self.bufferp == rffi.ptradd(rffi.cast(rffi.CCHARP, rawmem), + self.nb_bytes) + + # fill in the 'exchange_*' fields + self.fb_build_exchange(rawmem) + + # fill in the extra fields + self.fb_extra_fields(rawmem) + + # call libffi's ffi_prep_cif() function + res = jit_libffi.jit_ffi_prep_cif(rawmem) + if res != clibffi.FFI_OK: + runtime_error(u"libffi failed to build function type") + + return rawmem + + def get_item(self, nm): + (tp, offset) = self._type.get_desc(nm) + ptr = rffi.ptradd(self._buffer, offset) + return tp.ffi_load_from(ptr) + +# unicode utils + +from rpython.rlib.runicode import str_decode_utf_8, unicode_encode_utf_8 + +def unicode_from_utf8(s): + """Converts a `str` value to a `unicode` value assuming it's encoded in UTF8.""" + res, _ = str_decode_utf_8(s, len(s), 'strict') + return res + +def unicode_to_utf8(s): + """Converts a `unicode` value to a UTF8 encoded `str` value.""" + return unicode_encode_utf_8(s, len(s), 'strict') diff --git a/pixie/vm2/interpreter.py b/pixie/vm2/interpreter.py new file mode 100644 index 00000000..a6c274ef --- /dev/null +++ b/pixie/vm2/interpreter.py @@ -0,0 +1,914 @@ +from pixie.vm2.object import Object, Type, Continuation, stack_cons, runtime_error, affirm +from pixie.vm2.code import ns_registry, as_var +from pixie.vm2.primitives import nil, false +from pixie.vm2.array import Array +from pixie.vm2.arraymap import ArrayMap +import pixie.vm2.code as code +from pixie.vm2.keyword import Keyword, keyword +from pixie.vm2.code import BaseCode +import pixie.vm2.rt as rt +import rpython.rlib.jit as jit +import rpython.rlib.debug as debug +from pixie.vm2.debugger import expose + +kw_type = keyword(u"type") +kw_handler = keyword(u"handler") +kw_invoke = keyword(u"invoke") +kw_ast = keyword(u"ast") +kw_resolve_all = keyword(u"resolve-all") +kw_args = keyword(u"args") +kw_acc = keyword(u"acc") +kw_let = keyword(u"let") +kw_do = keyword(u"do") +kw_delimited_continuation = keyword(u"delimited-continuation") +kw_slice = keyword(u"slice") +kw_meta = keyword(u"meta") +kw_line = keyword(u"line") +kw_file = keyword(u"file") +kw_line_number = keyword(u"line-number") +kw_column_number = keyword(u"column-number") + +class Args(Object): + _type = Type(u"pixie.interpreter.Args") + _immutable_fields = ["_c_list[*]"] + + def type(self): + return Args._type + + def __init__(self, lst): + self._c_list = debug.make_sure_not_resized(lst) + + def c_array_val(self): + return self._c_list + + + + + +class AST(Object): + _immutable_fields_ = ["_c_meta"] + def __init__(self, meta): + self._c_meta = meta + + def get_short_location(self): + if self._c_meta != nil: + return self._c_meta.get_short_location() + return "" + + def get_long_location(self): + if self._c_meta != nil: + return self._c_meta.get_long_location() + return "" + + def describe(self): + if self._c_meta != nil: + return ArrayMap([kw_type, kw_ast, kw_meta, self._c_meta.describe()]) + +@as_var("pixie.ast.internal", "->Meta") +def new_meta(line, file, line_number, column_number): + return Meta((line.get_name(), file.get_name(), line_number.int_val()), column_number.int_val()) + +class Meta(Object): + _type = Type(u"pixie.stdlib.Meta") + _immutable_fields_ = ["_c_column_number", "_c_line_tuple"] + def __init__(self, line_tuple, column_number): + self._c_column_number = column_number + self._c_line_tuple = line_tuple + + def get_short_location(self): + (line, file, line_number) = self._c_line_tuple + + cl = self._c_column_number - 1 + assert cl >= 0 + return str(file) + " @ " + str(line[:cl]) + "^" + str(line[cl:]) + + def get_long_location(self): + (line, file, line_number) = self._c_line_tuple + ld = [] + ld.append(str(line)) + ld.append(" in ") + ld.append(str(file)) + ld.append(" at ") + ld.append(str(line_number)+":"+str(self._c_column_number)) + ld.append("\n") + for x in range(self._c_column_number - 1): + ld.append(" ") + ld.append("^\n") + + return "".join(ld) + + def describe(self): + (line, file, line_number) = self._c_line_tuple + return ArrayMap([kw_type, kw_meta, + kw_line, rt.wrap(line), + kw_file, rt.wrap(file), + kw_line_number, rt.wrap(line_number), + kw_column_number, rt.wrap(self._c_column_number)]) + + + +class PrevASTNil(AST): + def __init__(self): + AST.__init__(self, nil) + +@expose("_c_ast", "_c_locals") +class InterpretK(Continuation): + _immutable_ = True + def __init__(self, ast, locals): + self._c_ast = ast + self._c_locals = locals + + def call_continuation(self, val, stack): + ast = self._c_ast + return ast.interpret(val, self._c_locals, stack) + + def get_ast(self): + return self._c_ast + +@as_var("pixie.ast.internal", "->Const") +def new_const(val, meta): + return Const(val, meta) + +@expose("_c_val") +class Const(AST): + _immutable_fields_ = ["_c_val"] + _type = Type(u"pixie.interpreter.Const") + + def type(self): + return Const._type + + def __init__(self, val, meta=nil): + AST.__init__(self, meta) + self._c_val = val + + def interpret(self, _, locals, stack): + return self._c_val, stack + + def gather_locals(self): + return {} + + +@expose("_c_value", "_c_name", "_c_next") +class Locals(object): + _immutable_ = True + def __init__(self, name, value, next): + assert isinstance(name, Keyword) + self._c_value = value + self._c_name = name + self._c_next = next + + + + @staticmethod + @jit.unroll_safe + def get_local(self, name): + if self is None: + return nil + c = self + while c._c_name is not name: + c = c._c_next + if c is None: + runtime_error(u"Local" + name.to_repr() + u" is undefined") + return c._c_value + +@as_var("pixie.ast.internal", "->Lookup") +def new_lookup(name, meta): + return Lookup(name, meta) + +class Lookup(AST): + _immutable_fields_ = ["_c_name"] + _type = Type(u"pixie.ast-internal.Lookup") + + def type(self): + return VDeref._type + + def __init__(self, name, meta=nil): + assert isinstance(name, Keyword) + AST.__init__(self, meta) + self._c_name = name + + def interpret(self, _, locals, stack): + return Locals.get_local(locals, self._c_name), stack + + def gather_locals(self): + return {self._c_name: self._c_name} + +@as_var("pixie.ast.internal", "->Fn") +def new_fn(name, args, body, meta): + return Fn(name, args.array_val(), body, meta=meta) + +class Fn(AST): + _immutable_fields_ = ["_c_name", "_c_args", "_c_body", "_c_closed_overs"] + + _type = Type(u"pixie.ast.internal.Fn") + def type(self): + return Fn._type + + def __init__(self, name, args, body, closed_overs=[], meta=nil): + AST.__init__(self, meta) + self._c_name = name + self._c_args = args + self._c_body = body + + glocals = self._c_body.gather_locals() + for x in self._c_args: + if x in glocals: + del glocals[x] + + if self._c_name in glocals: + del glocals[self._c_name] + + + closed_overs = [None] * len(glocals) + idx = 0 + for k in glocals: + closed_overs[idx] = glocals[k] + idx += 1 + + self._c_closed_overs = closed_overs + + def gather_locals(self): + glocals = {} + + for x in self._c_closed_overs: + glocals[x] = x + + return glocals + + + @jit.unroll_safe + def interpret(self, _, locals, stack): + locals_prefix = None + for n in self._c_closed_overs: + locals_prefix = Locals(n, Locals.get_local(locals, n), locals_prefix) + + return InterpretedFn(self._c_name, self._c_args, locals_prefix, self._c_body, self), stack + +class InterpretedFn(code.BaseCode): + _immutable_fields_ = ["_c_arg_names[*]", "_c_locals", "_c_fn_ast", "_c_name", "_c_fn_def_ast"] + _type = Type(u"pixie.stdlib.InterpretedFn") + + def type(self): + return InterpretedFn._type + + def to_repr(self): + return u"" + + def to_str(self): + return u"" + + def __init__(self, name, arg_names, locals_prefix, ast, fn_def_ast): + assert isinstance(name, Keyword) + code.BaseCode.__init__(self) + self._c_arg_names = arg_names + self._c_name = name + if name is not nil: + self._c_locals = Locals(name, self, locals_prefix) + else: + self._c_locals = locals_prefix + self._c_fn_ast = ast + self._c_fn_def_ast = fn_def_ast + + def invoke_k(self, args, stack): + return self.invoke_k_with(args, stack, self) + + @jit.unroll_safe + def invoke_k_with(self, args, stack, self_fn): + # TODO: Check arg count + locals = self._c_locals + locals = Locals(self._c_name, self_fn, locals) + + if not len(args) == len(self._c_arg_names): + + runtime_error(u"Wrong number args to" + + self.to_repr() + + u", got " + + unicode(str(len(args))) + + u" expected " + unicode(str(len(self._c_arg_names))) + + u"\n" + unicode(self._c_fn_def_ast.get_long_location()), + u"pixie.stdlib.ArityException") + + for idx in range(len(self._c_arg_names)): + locals = Locals(self._c_arg_names[idx], args[idx], locals) + + return nil, stack_cons(stack, InterpretK(jit.promote(self._c_fn_ast), locals)) + + +@as_var("pixie.ast.internal", "->Invoke") +def new_invoke(args, meta): + return Invoke(args.array_val(), meta) + +class Invoke(AST): + _immutable_fields_ = ["_c_args[*]"] + _type = Type(u"pixie.ast-internal.Invoke") + + def type(self): + return Invoke._type + + def __init__(self, args, meta=nil): + AST.__init__(self, meta) + self._c_args = args + + def gather_locals(self): + glocals = {} + for x in self._c_args: + glocals = merge_dicts(glocals, x.gather_locals()) + + return glocals + + def interpret(self, _, locals, stack): + stack = stack_cons(stack, InvokeK(self)) + stack = stack_cons(stack, ResolveAllK(self, locals, [])) + stack = stack_cons(stack, InterpretK(self._c_args[0], locals)) + return nil, stack + + @jit.elidable_promote() + def arg_count(self): + return len(self._c_args) + + @jit.elidable_promote() + def get_arg(self, idx): + return self._c_args[idx] + + +class InvokeK(Continuation): + _immutable_ = True + def __init__(self, ast): + self._c_ast = ast + + def call_continuation(self, val, stack): + assert isinstance(val, Args) + fn = val._c_list[0] + args = val._c_list[1:] + return fn.invoke_k(args, stack) + + def get_ast(self): + return self._c_ast + + def describe(self): + return ArrayMap([kw_type, kw_invoke, kw_ast, self._c_ast]) + + + + +@as_var("pixie.ast.internal", "->Recur") +def new_invoke(args, meta): + return Recur(args.array_val(), meta) + +class Recur(Invoke): + _immutable_fields_ = ["_c_args", "_c_fn"] + def __init__(self, args, meta=nil): + Invoke.__init__(self, args, meta) + self._c_args = args + + def interpret(self, _, locals, stack): + stack = stack_cons(stack, RecurK(self)) + stack = stack_cons(stack, ResolveAllK(self, locals, [])) + stack = stack_cons(stack, InterpretK(self._c_args[0], locals)) + return nil, stack + +class RecurK(InvokeK): + _immutable_ = True + should_enter_jit = True + def __init__(self, ast): + InvokeK.__init__(self, ast) + + def get_ast(self): + return self._c_ast + + @jit.unroll_safe + def recur(self, args, stack): + fn = args.get_arg(0) + new_args = [None] * (args.get_arg_count() - 1) + for x in range(args.get_arg_count() - 1): + new_args[x] = args.get_arg(x + 1) + + return fn.invoke_k(new_args, stack) + +class ResolveAllK(Continuation): + _immutable_ = True + def __init__(self, args, locals, acc): + self._c_args_ast = args + self._c_locals = locals + self._c_acc = acc + + @jit.unroll_safe + def append_to_acc(self, val): + acc = [None] * (len(self._c_acc) + 1) + for x in range(len(self._c_acc)): + acc[x] = self._c_acc[x] + acc[len(self._c_acc)] = val + return acc + + def call_continuation(self, val, stack): + if len(self._c_acc) + 1 < self._c_args_ast.arg_count(): + stack = stack_cons(stack, ResolveAllK(self._c_args_ast, self._c_locals, self.append_to_acc(val))) + stack = stack_cons(stack, InterpretK(self._c_args_ast.get_arg(len(self._c_acc) + 1), self._c_locals)) + else: + return Args(self.append_to_acc(val)), stack + + return nil, stack + + def get_ast(self): + return self._c_args_ast.get_arg(len(self._c_acc)) + + def describe(self): + return ArrayMap([kw_type, kw_resolve_all, + kw_args, self._c_args_ast, + #kw_locals, Array(self._c_locals), + kw_acc, Array(self._c_acc), + kw_ast, self._c_args_ast.get_arg(len(self._c_acc))]) + +@as_var("pixie.ast.internal", "->Let") +def new_let(names, bindings, body, meta): + return Let(names.array_val(), bindings.array_val(), body, meta) + +class Let(AST): + _immutable_fields_ = ["_c_names", "_c_bindings", "_c_body"] + _type = Type(u"pixie.ast-internal.Let") + + def type(self): + return VDeref._type + + def __init__(self, names, bindings, body, meta=nil): + AST.__init__(self, meta) + self._c_names = names + self._c_bindings = bindings + self._c_body = body + + def gather_locals(self): + glocals = {} + for x in self._c_bindings: + glocals = merge_dicts(glocals, x.gather_locals()) + + glocals = merge_dicts(glocals, self._c_body.gather_locals()) + + for x in self._c_names: + if x in glocals: + del glocals[x] + + return glocals + + def interpret(self, _, locals, stack): + stack = stack_cons(stack, LetK(self, 0, locals)) + stack = stack_cons(stack, InterpretK(self._c_bindings[0], locals)) + return nil, stack + +class LetK(Continuation): + _immutable_ = True + def __init__(self, ast, idx, locals): + self._c_idx = idx + self._c_ast = ast + self._c_locals = locals + + def call_continuation(self, val, stack): + assert isinstance(self._c_ast, Let) + new_locals = Locals(self._c_ast._c_names[self._c_idx], val, self._c_locals) + + if self._c_idx + 1 < len(self._c_ast._c_names): + stack = stack_cons(stack, LetK(self._c_ast, self._c_idx + 1, new_locals)) + stack = stack_cons(stack, InterpretK(self._c_ast._c_bindings[self._c_idx + 1], new_locals)) + else: + stack = stack_cons(stack, InterpretK(self._c_ast._c_body, new_locals)) + + return nil, stack + + + def get_ast(self): + return self._c_ast + + def describe(self): + return ArrayMap([kw_type, kw_let, kw_ast, self._c_ast]) + +@as_var("pixie.ast.internal", "->If") +def new_if(test, then, els, meta): + return If(test, then, els, meta) + +class If(AST): + _immutable_fields_ = ["_c_test", "_c_then", "_c_else"] + _type = Type(u"pixie.ast-internal.If") + + def type(self): + return If._type + + def __init__(self, test, then, els, meta=nil): + AST.__init__(self, meta) + self._c_test = test + self._c_then = then + self._c_else = els + + def interpret(self, val, locals, stack): + stack = stack_cons(stack, IfK(self, locals)) + stack = stack_cons(stack, InterpretK(self._c_test, locals)) + return nil, stack + + def gather_locals(self): + glocals = self._c_test.gather_locals() + glocals = merge_dicts(glocals, self._c_then.gather_locals()) + glocals = merge_dicts(glocals, self._c_else.gather_locals()) + return glocals + +class IfK(Continuation): + _immutable_ = True + def __init__(self, ast, locals): + self._c_ast = ast + self._c_locals = locals + + def call_continuation(self, val, stack): + ast = self._c_ast + assert isinstance(self._c_ast, If) + if val is nil or val is false: + stack = stack_cons(stack, InterpretK(ast._c_else, self._c_locals)) + else: + stack = stack_cons(stack, InterpretK(ast._c_then, self._c_locals)) + return nil, stack + + def get_ast(self): + return self._c_ast + +@as_var("pixie.ast.internal", "->Do") +def new_do(args, meta): + return Do(args.array_val(), meta) + +class Do(AST): + _immutable_fields_ = ["_c_body_asts[*]"] + _type = Type(u"pixie.ast-internal.Do") + + def type(self): + return Do._type + + def __init__(self, args, meta=nil): + AST.__init__(self, meta) + self._c_body_asts = args + + @jit.unroll_safe + def interpret(self, val, locals, stack): + return nil, stack_cons(stack, DoK(self, self._c_body_asts, locals)) + + def gather_locals(self): + glocals = {} + for x in self._c_body_asts: + glocals = merge_dicts(glocals, x.gather_locals()) + + return glocals + + +class DoK(Continuation): + _immutable_ = True + def __init__(self, ast, do_asts, locals, idx=0): + self._c_ast = ast + self._c_locals = locals + self._c_body_asts = do_asts + self._c_idx = idx + + def call_continuation(self, val, stack): + if self._c_idx + 1 < len(self._c_body_asts): + stack = stack_cons(stack, DoK(self._c_ast, self._c_body_asts, self._c_locals, self._c_idx + 1)) + stack = stack_cons(stack, InterpretK(self._c_body_asts[self._c_idx], self._c_locals)) + return nil, stack + else: + stack = stack_cons(stack, InterpretK(self._c_body_asts[self._c_idx], self._c_locals)) + return nil, stack + + def get_ast(self): + return self._c_ast + + def describe(self): + return ArrayMap([kw_type, kw_do, kw_ast, self._c_ast]) + +@as_var("pixie.ast.internal", "->VDeref") +def new_vderef(in_ns, var_name, meta): + return VDeref(in_ns.get_name(), var_name, meta) + +dynamic_var_get = code.intern_var(u"pixie.stdlib", u"-dynamic-var-get") +dynamic_var_handler = code.intern_var(u"pixie.stdlib", u"dynamic-var-handler") + +@expose("_c_var") +class VDeref(AST): + _immutable_fields_ = ["_c_in_ns", "_c_var_name", "_c_var_ns"] + _type = Type(u"pixie.ast-internal.VDeref") + + def type(self): + return VDeref._type + + def __init__(self, in_ns, var_name, meta=nil): + AST.__init__(self, meta) + self._c_in_ns = in_ns + self._c_var_ns = None if var_name.get_ns() == u"" else var_name.get_ns() + self._c_var_name = var_name.get_name() + + def interpret(self, val, locals, stack): + ns = ns_registry.find_or_make(self._c_in_ns) + if ns is None: + runtime_error(u"Namespace " + self._c_in_ns + u" is not found", + u"pixie.stdlib/UndefinedNamespace") + + var = ns.resolve_in_ns_ex(self._c_var_ns, self._c_var_name) + if var is not None: + if var.is_dynamic(): + return dynamic_var_get.invoke_k([dynamic_var_handler.deref(), var], stack) + else: + return var.deref(), stack + else: + runtime_error(u"Var " + + (self._c_var_ns + u"/" if self._c_var_ns else u"") + + self._c_var_name + + u" is undefined in " + + self._c_in_ns, + u"pixie.stdlib/UndefinedVar") + + def gather_locals(self): + return {} + + +@as_var("pixie.ast.internal", "->VarConst") +def new_vderef(in_ns, var_name, meta): + return VarConst(in_ns.get_name(), var_name, meta) + +@expose("_c_var") +class VarConst(AST): + _immutable_fields_ = ["_c_in_ns", "_c_var_name"] + _type = Type(u"pixie.ast.internal.VarConst") + + def type(self): + return VarConst._type + + def __init__(self, in_ns, var_name, meta=nil): + AST.__init__(self, meta) + self._c_in_ns = in_ns + self._c_var_ns = None if var_name.get_ns() == u"" else var_name.get_ns() + self._c_var_name = var_name.get_name() + + def interpret(self, val, locals, stack): + ns = ns_registry.find_or_make(self._c_in_ns) + if ns is None: + runtime_error(u"Namespace " + self._c_in_ns + u" is not found", + u"pixie.stdlib/UndefinedNamespace") + + var = ns.resolve_in_ns_ex(self._c_var_ns, self._c_var_name) + if var is not None: + return var, stack + else: + if self._c_var_ns is not None: + runtime_error(u"Var " + + (self._c_var_ns + u"/" if self._c_var_ns else u"") + + self._c_var_name + + u" is undefined in " + + self._c_in_ns, + u"pixie.stdlib/UndefinedVar") + else: + return ns.intern_or_make(self._c_var_name), stack + + + def gather_locals(self): + return {} + + + +from rpython.rlib.jit import JitDriver +from rpython.rlib.objectmodel import we_are_translated +def get_printable_location(ast): + return ast.get_short_location() + +class VirtArgs(object): + _virtualizable_ = ["_args[*]"] + def __init__(self, args): + self = jit.hint(self, access_directly=True, fresh_virtualizable=True) + self._args = debug.make_sure_not_resized(args) + + def get_arg_count(self): + return len(self._args) + + def get_arg(self, idx): + assert 0 <= idx < len(self._args) + return self._args[idx] + +jitdriver = JitDriver(greens=["ast"], reds=["stack", "val", "args", "cont"], get_printable_location=get_printable_location, + virtualizables=["args"]) +jit.set_param(jitdriver, "trace_limit", 30000) + +throw_var = code.intern_var(u"pixie.stdlib", u"throw") + +def run_stack(val, cont, stack=None, enter_debug=True): + stack = None + val = None + ast = cont.get_ast() + args = None + while True: + jitdriver.jit_merge_point(ast=ast, stack=stack, val=val, args=args, cont=cont) + try: + if isinstance(cont, RecurK): + val, stack = cont.recur(args, stack) + args = None + else: + val, stack = cont.call_continuation(val, stack) + ast = cont.get_ast() + except SystemExit: + raise + except BaseException as ex: + #if enter_debug: + # from pixie.vm2.debugger import debug + #debug(cont, stack, val) + #print_stacktrace(cont, stack) + if not we_are_translated(): + import traceback + print(traceback.format_exc()) + print ex + + val, stack = throw_var.invoke_k([keyword(u"pixie.stdlib/InternalException"), + keyword(u"TODO")], stack_cons(stack, cont)) + + if stack: + cont = stack._cont + stack = stack._parent + else: + break + + if isinstance(cont, RecurK): + args = VirtArgs(val.c_array_val()) + val = None + jitdriver.can_enter_jit(ast=ast, stack=stack, val=val, args=args, cont=cont) + + + return val + +def stacktrace_for_cont(cont): + ast = cont.get_ast() + if ast: + return ast.get_long_location() + +def print_stacktrace(cont, stack): + st = [] + st.append(stacktrace_for_cont(cont)) + while stack: + st.append(stacktrace_for_cont(stack._cont)) + stack = stack._parent + st.reverse() + for line in st: + print line + + + +## Handler code + + +class WithHandler(code.BaseCode): + def __init__(self): + BaseCode.__init__(self) + + def invoke_k(self, args, stack): + affirm(len(args) == 2, u"-with-handler takes one argument") + handler = args[0] + fn = args[1] + + stack = stack_cons(stack, Handler(handler)) + val, stack = fn.invoke_k([], stack) + return val, stack + + + +class Handler(Continuation): + _immutable_ = True + def __init__(self, handler): + self._handler = handler + + def handler(self): + return self._handler + + def get_ast(self): + return None + + def call_continuation(self, val, stack): + return val, stack + + def describe(self): + return ArrayMap([kw_type, kw_handler, kw_handler, self._handler]) + +class DelimitedContinuation(code.NativeFn): + _immutable_fields_ = ["_slice[*]"] + _type = Type(u"pixie.stdlib.DelmitedContinuation") + + def type(self): + return DelimitedContinuation._type + + def __init__(self, slice): + self._slice = slice + + def describe(self): + arr = [None] * len(self._slice) + for idx, x in enumerate(self._slice): + arr[idx] = x.describe() + + return ArrayMap([kw_type, kw_delimited_continuation, + kw_slice, Array(arr)]) + + @jit.unroll_safe + def invoke_k(self, args, stack): + affirm(len(args) == 1, u"Delmited continuations only take one argument") + for x in range(len(self._slice)): + stack = stack_cons(stack, self._slice[x]) + return args[0], stack + +@as_var(u"pixie.stdlib", u"describe-internal-object") +def describe_delimited_continuation(k): + return k.describe() + +class EffectFunction(code.BaseCode): + _type = Type(u"pixie.stdlib.EffectFn") + + def type(self): + return EffectFunction._type + + def __init__(self, inner_fn): + BaseCode.__init__(self) + self._inner_fn = inner_fn + + def invoke_k(self, args, stack): + affirm(len(args) >= 1, u"Effect functions require at least one argument") + handler = args[0] + + stack, slice, handler = self.slice_stack(stack, handler) + + new_args = [None] * (len(args) + 1) + + # Patch in the handler because the original handler may be nil + new_args[0] = handler + for x in range(1, len(args)): + new_args[x] = args[x] + new_args[len(args)] = DelimitedContinuation(slice) + + + return self._inner_fn.invoke_k(new_args, stack) + + + @jit.unroll_safe + def append_args(self, args, val): + new_args = [None] * (len(args) + 1) + for x in range(len(args)): + new_args[x] = args[x] + + new_args[(len(args))] = val + + return new_args + + @jit.unroll_safe + def slice_stack(self, orig_stack, handler): + if handler is nil: + affirm(isinstance(self._inner_fn, code.PolymorphicFn), u"Can't dispatch with a nil handler from a non polymorphic effect") + + size = 0 + stack = orig_stack + ret_handler = None + + while True: + if stack is None: + ## No hander found + if not we_are_translated(): + print "Looking for handler, none found, ", handler + + raise SystemExit() + + if (isinstance(stack._cont, Handler) and stack._cont.handler() is handler) or \ + (handler is nil and isinstance(stack._cont, Handler) and self._inner_fn.satisfied_by(stack._cont.handler())): + size += 1 + ret_handler = stack._cont.handler() + break + + size += 1 + stack = stack._parent + + slice = [None] * size + + stack = orig_stack + for x in range(size - 1, -1, -1): + slice[x] = stack._cont + stack = stack._parent + + return stack, slice, ret_handler + + +def merge_dicts(a, b): + z = a.copy() + z.update(b) + return z + + +# Eval support + +class EvalFn(code.NativeFn): + + _type = Type(u"pixie.stdlib.EvalFn") + + def type(self): + return EvalFn._type + + + @jit.unroll_safe + def invoke_k(self, args, stack): + affirm(len(args) == 1, u"Eval takes one argument") + ast = args[0] + + result = ast.interpret(nil, None, stack) + return result + +as_var("pixie.ast.internal", "eval")(EvalFn()) diff --git a/pixie/vm2/jit_tables.py b/pixie/vm2/jit_tables.py new file mode 100644 index 00000000..bce16cef --- /dev/null +++ b/pixie/vm2/jit_tables.py @@ -0,0 +1,67 @@ +from pixie.vm2.object import Type, Object, affirm +from pixie.vm2.code import as_var +import rpython.rlib.jit as jit +from pixie.vm2.primitives import nil, true, false +import pixie.vm2.rt as rt + + +class SwitchTable(Object): + _type = Type(u"pixie.stdlib.SwitchTable") + _immutable_fields_ = ["_switch_table"] + + def type(self): + return SwitchTable._type + + def __init__(self, d): + self._switch_table = d + + + @jit.elidable_promote() + def lookup(self, itm): + return self._switch_table.get(itm, nil) + + def invoke_k(self, args, stack): + affirm(len(args) == 1, u"SwitchTables should be called with one arg") + return self.lookup(args[0]), stack + + + +@as_var("switch-table") +def swith_table__args(args): + idx = 0 + acc = {} + while idx < len(args): + affirm(idx + 1 < len(args), u"Even number of args should be passed to switch-table") + acc[args[idx]] = args[idx + 1] + idx += 2 + + return SwitchTable(acc) + +class ContainsTable(Object): + _type = Type(u"pixie.stdlib.ContainsTable") + _immutable_fields_ = ["_switch_table"] + + def type(self): + return ContainsTable._type + + def __init__(self, dict): + self._switch_table = dict + + + @jit.elidable_promote() + def lookup(self, itm): + return rt.wrap(itm in self._switch_table) + + def invoke_k(self, args, stack): + affirm(len(args) == 1, u"ContainsTables should be called with one arg") + return self.lookup(args[0]), stack + +@as_var("contains-table") +def contains_table__args(args): + idx = 0 + acc = {} + while idx < len(args): + acc[args[idx]] = args[idx] + idx += 1 + + return ContainsTable(acc) \ No newline at end of file diff --git a/pixie/vm2/keyword.py b/pixie/vm2/keyword.py new file mode 100644 index 00000000..34956b8c --- /dev/null +++ b/pixie/vm2/keyword.py @@ -0,0 +1,105 @@ +from pixie.vm2.object import Object, Type +from pixie.vm2.primitives import nil +from pixie.vm2.string import String +#import pixie.vm.stdlib as proto +from pixie.vm2.code import extend, as_var +import pixie.vm2.rt as rt +#import pixie.vm.util as util +from rpython.rlib.rarithmetic import intmask, r_uint + + +class Keyword(Object): + _type = Type(u"pixie.stdlib.Keyword") + def __init__(self, name): + self._str = name + self._w_name = None + self._w_ns = None + self._hash = r_uint(0) + + def type(self): + return Keyword._type + + def init_names(self): + if self._w_name is None: + s = self._str.split(u"/") + if len(s) == 2: + self._w_ns = rt.wrap(s[0]) + self._w_name = rt.wrap(s[1]) + elif len(s) == 1: + self._w_name = rt.wrap(s[0]) + self._w_ns = nil + else: + self._w_ns = rt.wrap(s[0]) + self._w_name = rt.wrap(u"/".join(s[1:])) + + def __repr__(self): + return u":" + self._str + + def to_str(self): + return u":" + self._str + + def to_repr(self): + return u":" + self._str + + def store_hash(self, hash): + self._hash = hash + + def get_name(self): + self.init_names() + return rt.unwrap_string(self._w_name) + + def get_ns(self): + self.init_names() + return rt.unwrap_string(self._w_ns) + + def get_hash(self): + return self._hash + + + +class KeywordCache(object): + def __init__(self): + self._cache = {} + + def intern(self, nm): + kw = self._cache.get(nm, None) + + if kw is None: + kw = Keyword(nm) + self._cache[nm] = kw + + return kw + +_kw_cache = KeywordCache() + +def keyword(nm, ns=None): + if ns: + nm = u"/".join([ns, nm]) + return _kw_cache.intern(nm) + +# +# @extend(proto._name, Keyword) +# def _name(self): +# assert isinstance(self, Keyword) +# self.init_names() +# return self._w_name +# +# @extend(proto._namespace, Keyword) +# def _namespace(self): +# assert isinstance(self, Keyword) +# self.init_names() +# return self._w_ns +# +# @extend(proto._hash, Keyword) +# def _hash(self): +# assert isinstance(self, Keyword) +# if self._hash == 0: +# self._hash = util.hash_unencoded_chars(self._str) +# return rt.wrap(intmask(self._hash)) +# +@as_var("keyword") +def _keyword(s): + if not isinstance(s, String): + from pixie.vm2.object import runtime_error + runtime_error(u"Symbol name must be a string") + return keyword(s._str) diff --git a/pixie/vm2/numbers.py b/pixie/vm2/numbers.py new file mode 100644 index 00000000..076488ab --- /dev/null +++ b/pixie/vm2/numbers.py @@ -0,0 +1,526 @@ +import pixie.vm2.object as object +from pixie.vm2.object import affirm +from pixie.vm2.primitives import true, false +from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rbigint import rbigint +import rpython.rlib.jit as jit +from pixie.vm2.code import DoublePolymorphicFn, extend, Protocol, as_var, wrap_fn, munge +#from pixie.vm.libs.pxic.util import add_marshall_handlers +import pixie.vm2.rt as rt + +import math + +class Number(object.Object): + _type = object.Type(u"pixie.stdlib.Number") + + def type(self): + return Number._type + + + +class Integer(Number): + _type = object.Type(u"pixie.stdlib.Integer", Number._type) + _immutable_fields_ = ["_int_val"] + + def __init__(self, i_val): + self._int_val = i_val + + def int_val(self): + return self._int_val + + def r_uint_val(self): + return r_uint(self._int_val) + + def type(self): + return Integer._type + + def to_str(self): + return unicode(str(self._int_val)) + + def to_repr(self): + return unicode(str(self._int_val)) + +class SizeT(Integer): + _type = object.Type(u"pixie.stdlib.SizeT", Integer._type) + _immutable_fields_ = ["_ruint_val"] + + def __init__(self, val): + self._ruint_val = r_uint(val) + + def r_uint_val(self): + return self._ruint_val + + def int_val(self): + return intmask(self._ruint_val) + + def type(self): + return self._type + + def to_str(self): + return unicode(str(self._ruint_val)) + + def to_repr(self): + return unicode(str(self._ruint_val)) + +zero_int = Integer(0) +one_int = Integer(1) + +class BigInteger(Number): + _type = object.Type(u"pixie.stdlib.BigInteger", Number._type) + _immutable_fields_ = ["_bigint_val"] + + def __init__(self, bi_val): + self._bigint_val = bi_val + + def bigint_val(self): + return self._bigint_val + + def type(self): + return BigInteger._type + +class Float(Number): + _type = object.Type(u"pixie.stdlib.Float", Number._type) + _immutable_fields_ = ["_float_val"] + + def __init__(self, f_val): + self._float_val = f_val + + def float_val(self): + return self._float_val + + def type(self): + return Float._type + +class Ratio(Number): + _type = object.Type(u"pixie.stdlib.Ratio", Number._type) + _immutable_fields_ = ["_numerator", "_denominator"] + + def __init__(self, numerator, denominator): + assert numerator is not None and denominator is not None + self._numerator = numerator + self._denominator = denominator + + def numerator(self): + return self._numerator + + def denominator(self): + return self._denominator + + def type(self): + return Ratio._type + +@wrap_fn +def ratio_write(obj): + assert isinstance(obj, Ratio) + return rt.vector(rt.wrap(obj.numerator()), rt.wrap(obj.denominator())) + +@wrap_fn +def ratio_read(obj): + return Ratio(rt.nth(obj, rt.wrap(0)).int_val(), rt.nth(obj, rt.wrap(1)).int_val()) + +#add_marshall_handlers(Ratio._type, ratio_write, ratio_read) + +IMath = as_var("IMath")(Protocol(u"IMath")) +_add = as_var("-add")(DoublePolymorphicFn(u"-add", IMath)) +_sub = as_var("-sub")(DoublePolymorphicFn(u"-sub", IMath)) +_mul = as_var("-mul")(DoublePolymorphicFn(u"-mul", IMath)) +_div = as_var("-div")(DoublePolymorphicFn(u"-div", IMath)) +_quot = as_var("-quot")(DoublePolymorphicFn(u"-quot", IMath)) +_rem = as_var("-rem")(DoublePolymorphicFn(u"-rem", IMath)) +_lt = as_var("-lt")(DoublePolymorphicFn(u"-lt", IMath)) +_gt = as_var("-gt")(DoublePolymorphicFn(u"-gt", IMath)) +_lte = as_var("-lte")(DoublePolymorphicFn(u"-lte", IMath)) +_gte = as_var("-gte")(DoublePolymorphicFn(u"-gte", IMath)) +_num_eq = as_var("-num-eq")(DoublePolymorphicFn(u"-num-eq", IMath)) +_num_eq.set_default_fn(wrap_fn(lambda a, b: false)) + +#as_var("MAX-NUMBER")(Integer(100000)) # TODO: set this to a real max number + + +# Ordering of conversions. If a given function is called with two numbers of different +# types, then the type lower on this list will always be upconverted before the opration is +# performed. + +number_orderings = [Integer, SizeT, BigInteger, Float] + +# Given a value of a certain type, how do we convert it to the *primitive* of another type? +# Notice that conversions for a type to itself must exist, this is then the simple upwrap case. +conversion_templates = {Integer: {Integer: "{x}.int_val()", + SizeT: "{x}.r_uint_val()", + Float: "float({x}.int_val())", + BigInteger: "rbigint.fromint({x}.int_val())"}, + SizeT: {SizeT:"{x}.r_uint_val()", + Float:"float({x}.int_val())", + BigInteger:"rbigint.fromint({x}.int_val())"}, + BigInteger: {BigInteger: "{x}.bigint_val()", + Float: "{x}.bigint_val().tofloat()"}, + Float: {Float: "{x}.float_val()"}} + +# Given an operation and a type, how do we perform that operation? + +operations = {"-add": {Integer: "{x} + {y}", + SizeT: "{x} + {y}", + BigInteger: "{x}.add({y})", + Float: "{x} + {y}"}, + "-sub": {Integer: "{x} - {y}", + SizeT: "{x} - {y}", + BigInteger: "{x}.sub({y})", + Float: "{x} - {y}"}, + "-mul": {Integer: "{x} * {y}", + SizeT: "{x} * {y}", + BigInteger: "{x}.mul({y})", + Float: "{x} * {y}"}, + "-div": {Integer: "{x} / {y}", + SizeT: "{x} / {y}", + BigInteger: "{x}.div({y})", + Float: "{x} / {y}"}, + "-rem": {Integer: "{x} % {y}", + SizeT: "{x} % {y}", + BigInteger: "{x}.mod({y})", + Float: "math.fmod({x}, {y})"}, + "-gt": {Integer: "{x} > {y}", + SizeT: "{x} > {y}", + BigInteger: "{x}.gt({y})", + Float: "{x} > {y}"}, + + "-lt": {Integer: "{x} < {y}", + SizeT: "{x} < {y}", + BigInteger: "{x}.lt({y})", + Float: "{x} < {y}"}, + + "-gte": {Integer: "{x} >= {y}", + SizeT: "{x} >= {y}", + BigInteger: "{x}.ge({y})", + Float: "{x} >= {y}"}, + + "-lte": {Integer: "{x} <= {y}", + SizeT: "{x} <= {y}", + BigInteger: "{x}.le({y})", + Float: "{x} <= {y}"}, + + "-num-eq": {Integer: "{x} == {y}", + SizeT: "{x} == {y}", + BigInteger: "{x}.eq({y})", + Float: "{x} == {y}"}, + } + +# These functions return bool and so should always be returned via rt.wrap +binop_names = {"-gt", "-lt", "-gte", "-lte", "-num-eq"} + +# How do we wrap primitives? +wrappers = {Integer: "rt.wrap({x})", + SizeT: "SizeT({x})", + BigInteger: "rt.wrap({x})", + Float: "rt.wrap({x})"} + + + +op_template = """ +@extend({pfn}, {t1}._type, {t2}._type) +def {pfn}_{t1}_{t2}(x, y): + return {result} +""" + +def get_rank(t1): + for idx, tp in enumerate(number_orderings): + if tp == t1: + return idx + + assert False, str(t1) + " not found" + + +def make_num_op(pfn, t1, t2): + t1rank = get_rank(t1) + t2rank = get_rank(t2) + + if t1rank >= t2rank: + t1_conv = t1 + t2_conv = t1 + else: + t1_conv = t2 + t2_conv = t2 + + wrapper = wrappers[t1_conv] if pfn not in binop_names else "rt.wrap({x})" + + result = wrapper.format(x=operations[pfn][t1_conv].format(x=conversion_templates[t1][t1_conv].format(x="x"), + y=conversion_templates[t2][t2_conv].format(x="y"))) + + templated = op_template.format(pfn=munge(pfn), + t1=str(t1._type._name.split(".")[-1]), + t2=str(t2._type._name.split(".")[-1]), + result=result) + + return templated + +for pfn in operations: + for t1 in number_orderings: + for t2 in number_orderings: + exec make_num_op(pfn, t1, t2) + + + +def gcd(u, v): + while v != 0: + r = u % v + u = v + v = r + return u + +@extend(_div, Integer._type, Integer._type) +def _div(n, d): + assert isinstance(n, Integer) and isinstance(d, Integer) + nv = n.int_val() + dv = d.int_val() + object.affirm(dv != 0, u"Divide by zero") + g = gcd(nv, dv) + if g == 0: + return rt.wrap(0) + nv = nv / g + dv = dv / g + if dv == 1: + return rt.wrap(nv) + elif dv == -1: + return rt.wrap(-1 * nv) + else: + if dv < 0: + nv = nv * -1 + dv = dv * -1 + return Ratio(nv, dv) + +# @extend(_add, Ratio._type, Ratio._type) +# def _add(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return rt._div(rt._add(rt.wrap(b.numerator() * a.denominator()), +# rt.wrap(a.numerator() * b.denominator())), +# rt.wrap(a.denominator() * b.denominator())) +# +# @extend(_sub, Ratio._type, Ratio._type) +# def _sub(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return rt._div(rt._add(rt.wrap(-1 * b.numerator() * a.denominator()), +# rt.wrap(a.numerator() * b.denominator())), +# rt.wrap(a.denominator() * b.denominator())) +# +# @extend(_mul, Ratio._type, Ratio._type) +# def _mul(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return rt._div(rt.wrap(b.numerator() * a.numerator()), +# rt.wrap(b.denominator() * a.denominator())) +# +# @extend(_div, Ratio._type, Ratio._type) +# def _div(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return rt._div(rt.wrap(b.denominator() * a.numerator()), +# rt.wrap(b.numerator() * a.denominator())) +# +# @extend(_quot, Ratio._type, Ratio._type) +# def _quot(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return rt.wrap((a.numerator() * b.denominator()) / (a.denominator() * b.numerator())) +# +# @extend(_rem, Ratio._type, Ratio._type) +# def _rem(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# q = rt.wrap((a.numerator() * b.denominator()) / (a.denominator() * b.numerator())) +# return rt._sub(a, rt._mul(q, b)) +# +# @extend(_lt, Ratio._type, Ratio._type) +# def _lt(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return true if a.numerator() * b.denominator() < b.numerator() * a.denominator() else false +# +# @extend(_gt, Ratio._type, Ratio._type) +# def _gt(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return rt._lt(b, a) +# +# @extend(_lte, Ratio._type, Ratio._type) +# def _lte(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return true if rt._lt(b, a) is false else false +# +# @extend(_gte, Ratio._type, Ratio._type) +# def gte(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return true if rt._lt(a, b) is false else false +# +# @extend(_num_eq, Ratio._type, Ratio._type) +# def _num_eq(a, b): +# assert isinstance(a, Ratio) and isinstance(b, Ratio) +# return true if a.numerator() == b.numerator() and a.denominator() == b.denominator() else false + +mixed_op_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) +def {pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + return rt.{pfn}({conv1}(a), {conv2}(b)) +""" + +def to_ratio(x): + if isinstance(x, Ratio): + return x + else: + return Ratio(x.int_val(), 1) + +def to_ratio_conv(c): + if c == Ratio: + return "" + else: + return "to_ratio" + +def to_float(x): + if isinstance(x, Float): + return x + if isinstance(x, Ratio): + return rt.wrap(x.numerator() / float(x.denominator())) + if isinstance(x, BigInteger): + return rt.wrap(x.bigint_val().tofloat()) + assert False + +def to_float_conv(c): + if c == Float: + return "" + else: + return "to_float" + +def define_mixed_ops(): + for (c1, c2) in [(Integer, Ratio), (Ratio, Integer)]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_ratio_conv(c1), conv2=to_ratio_conv(c2)) + exec code + + for (c1, c2) in [(Float, Ratio), (Ratio, Float)]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) + exec code + + for (c1, c2) in [(Float, BigInteger), (BigInteger, Float)]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) + exec code + +#define_mixed_ops() + +# def add(a, b): +# if isinstance(a, Integer): +# if isinstance(b, Integer): +# return Integer(a.int_val() + b.int_val()) +# +# raise Exception("Add error") + +def eq(a, b): + if isinstance(a, Integer): + if isinstance(b, Integer): + return true if a.int_val() == b.int_val() else false + + raise Exception("Add error") + + +def init(): + import pixie.vm2.stdlib as proto + from pixie.vm2.string import String + + # @extend(proto._str, Integer._type) + # def _str(i): + # return rt.wrap(unicode(str(i.int_val()))) + # + # @extend(proto._repr, Integer._type) + # def _repr(i): + # return rt.wrap(unicode(str(i.int_val()))) + # + # @extend(proto._str, BigInteger._type) + # def _str(b): + # return rt.wrap(unicode(b.bigint_val().format('0123456789', suffix='N'))) + # + # @extend(proto._repr, BigInteger._type) + # def _repr(b): + # return rt.wrap(unicode(b.bigint_val().format('0123456789', suffix='N'))) + # + # @extend(proto._str, Float._type) + # def _str(f): + # return rt.wrap(unicode(str(f.float_val()))) + # + # @extend(proto._repr, Float._type) + # def _repr(f): + # return rt.wrap(unicode(str(f.float_val()))) + # + # @extend(proto._repr, Ratio._type) + # def _repr(r): + # return rt.wrap(unicode(str(r.numerator()) + "/" + str(r.denominator()))) + # + # @extend(proto._str, Ratio._type) + # def _str(r): + # return rt.wrap(unicode(str(r.numerator()) + "/" + str(r.denominator()))) + # + # @as_var("numerator") + # def numerator(r): + # affirm(isinstance(r, Ratio), u"First argument must be a Ratio") + # return rt.wrap(r.numerator()) + # + # @as_var("denominator") + # def denominator(r): + # affirm(isinstance(r, Ratio), u"First argument must be a Ratio") + # return rt.wrap(r.denominator()) + +from rpython.rlib.rsre import rsre_re as re +# inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 +# sign hex oct radix decimal biginteger +# 1 2 3 4 5 6 7 8 +int_matcher = re.compile(u'^([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))(N)?$') + +float_matcher = re.compile(u'^([-+]?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+)?)$') +ratio_matcher = re.compile(u'^([-+]?[0-9]+)/([0-9]+)$') + +def parse_int(m): + sign = 1 + if m.group(1) == u'-': + sign = -1 + + radix = 10 + + if m.group(7): + num = m.group(7) + elif m.group(2): + radix = 16 + num = m.group(3) + elif m.group(4): + radix = 8 + num = m.group(4) + elif m.group(5): + radix = int(m.group(5)) + num = m.group(6) + else: + return None + + if m.group(8): + return rt.wrap(rbigint.fromstr(str(m.group(1) + num), radix)) + else: + return rt.wrap(sign * int(str(num), radix)) + +def parse_float(m): + return rt.wrap(float(str(m.group(0)))) + +def parse_ratio(m): + n = int(str(m.group(1))) + d = int(str(m.group(2))) + return Ratio(n, d) + +def parse_number(s): + m = int_matcher.match(s) + if m: + return parse_int(m) + else: + m = float_matcher.match(s) + if m: + return parse_float(m) + else: + m = ratio_matcher.match(s) + if m: + return parse_ratio(m) + else: + return None + +@as_var(u"-parse-number") +def _parse_number(x): + from pixie.vm2.string import String + assert isinstance(x, String) + return parse_number(x._str) \ No newline at end of file diff --git a/pixie/vm2/object.py b/pixie/vm2/object.py new file mode 100644 index 00000000..aa12f9a0 --- /dev/null +++ b/pixie/vm2/object.py @@ -0,0 +1,190 @@ +import rpython.rlib.jit as jit +from rpython.rlib.objectmodel import we_are_translated + +class Object(object): + """ Base Object for all VM objects + """ + _attrs_ = () + def type(self): + if not we_are_translated(): + print "FOR TYPE", self + affirm(False, u".type isn't overloaded") + + @jit.unroll_safe + def invoke_k(self, args, stack): + from pixie.vm2.code import intern_var + var = intern_var(u"pixie.stdlib", u"-invoke") + + new_args = [None] * (len(args) + 1) + + new_args[0] = self + for x in range(len(args)): + new_args[x + 1] = args[x] + + return var.deref().invoke_k(new_args, stack) + + def int_val(self): + affirm(False, u"Expected Number, not " + self.type().name()) + return 0 + + def r_uint_val(self): + affirm(False, u"Expected Number, not " + self.type().name()) + return 0 + + def hash(self): + import pixie.vm2.rt as rt + return rt.wrap(compute_identity_hash(self)) + + def promote(self): + return self + + def get_field(self, k, not_found): + runtime_error(u"Unsupported operation get-field") + + def to_str(self): + tp = self.type() + assert isinstance(tp, Type) + return u"" + + def to_repr(self): + tp = self.type() + assert isinstance(tp, Type) + return u"" + + + +class TypeRegistry(object): + def __init__(self): + self._types = {} + self._ns_registry = None + + def register_type(self, nm, tp): + if self._ns_registry is None: + self._types[nm] = tp + else: + self.var_for_type_and_name(nm, tp) + + def var_for_type_and_name(self, nm, tp): + splits = nm.split(u".") + size = len(splits) - 1 + assert size >= 0 + ns = u".".join(splits[:size]) + name = splits[size] + var = self._ns_registry.find_or_make(ns).intern_or_make(name) + var.set_root(tp) + return var + + def set_registry(self, registry): + self._ns_registry = registry + for nm in self._types: + tp = self._types[nm] + self.var_for_type_and_name(nm, tp) + + + def get_by_name(self, nm, default=None): + return self._types.get(nm, default) + + +_type_registry = TypeRegistry() + +def get_type_by_name(nm): + return _type_registry.get_by_name(nm) + +class Type(Object): + def __init__(self, name, parent=None, object_inited=True): + assert isinstance(name, unicode), u"Type names must be unicode" + _type_registry.register_type(name, self) + self._name = name + + if object_inited: + if parent is None: + parent = Object._type + + parent.add_subclass(self) + + self._parent = parent + self._subclasses = [] + + def name(self): + return self._name + + def type(self): + return Type._type + + def add_subclass(self, tp): + self._subclasses.append(tp) + + def subclasses(self): + return self._subclasses + + def to_str(self): + return u"" + + def to_repr(self): + return u"" + + +Object._type = Type(u"pixie.stdlib.Object", None, False) +Type._type = Type(u"pixie.stdlib.Type") + +@jit.elidable_promote() +def istypeinstance(obj_type, t): + if not isinstance(obj_type, Type): + return False + if obj_type is t: + return True + elif obj_type._parent is not None: + obj_type = obj_type._parent + while obj_type is not None: + if obj_type is t: + return True + obj_type = obj_type._parent + return False + else: + return False + +class Continuation(object): + should_enter_jit = False + _immutable_ = True + def call_continuation(self, val, stack): + assert False + return None, stack + + def get_ast(self): + from pixie.vm2.primitives import nil + return nil + + +class StackCell(object): + """Defines an immutable call stack, stacks can be copied, spliced and combined""" + _immutable_fields_ = ["_parent", "_cont"] + def __init__(self, cont, parent_stack): + self._parent = parent_stack + self._cont = cont + +def stack_cons(stack, other): + return StackCell(other, stack) + + + + +class RuntimeException(Object): + _type = Type(u"pixie.stdlib.RuntimeException") + def __init__(self, msg, kw): + self._msg = msg + self._kw = kw + +## TODO: fix +def affirm(f, msg): + if not f: + print msg + raise WrappedException(RuntimeException(msg, u"pixie.stdlib.AssertionException")) + +class WrappedException(BaseException): + def __init__(self, ex): + self._ex = ex + +def runtime_error(msg, kw=None): + print msg + raise WrappedException(RuntimeException(msg, kw)) + diff --git a/pixie/vm2/platform.py b/pixie/vm2/platform.py new file mode 100644 index 00000000..db285eb4 --- /dev/null +++ b/pixie/vm2/platform.py @@ -0,0 +1,23 @@ +from rpython.translator.platform import platform +from pixie.vm2.string import String +from pixie.vm2.code import as_var +from pixie.vm2.array import Array +from rpython.rlib.clibffi import get_libc_name +import os + + +as_var("pixie.platform", "os")(String(unicode(os.name))) +as_var("pixie.platform", "name")(String(unicode(platform.name))) +as_var("pixie.platform", "so-ext")(String(unicode(platform.so_ext))) +as_var("pixie.platform", "lib-c-name")(String(unicode(get_libc_name()))) + +c_flags = [] +for itm in platform.cflags: + c_flags.append(String(unicode(itm))) + +as_var("pixie.platform", "c-flags")(Array(c_flags)) + +link_flags = [] +for itm in platform.link_flags: + c_flags.append(String(unicode(itm))) +as_var("pixie.platform", "link-flags")(Array(link_flags)) diff --git a/pixie/vm2/primitives.py b/pixie/vm2/primitives.py new file mode 100644 index 00000000..b9645e4b --- /dev/null +++ b/pixie/vm2/primitives.py @@ -0,0 +1,25 @@ +import pixie.vm2.object as object + + +class Nil(object.Object): + _type = object.Type(u"pixie.stdlib.Nil") + + def __repr__(self): + return u"nil" + + def type(self): + return Nil._type + + +nil = Nil() + + +class Bool(object.Object): + _type = object.Type(u"pixie.stdlib.Bool") + + def type(self): + return Bool._type + + +true = Bool() +false = Bool() diff --git a/pixie/vm2/pxic_reader.py b/pixie/vm2/pxic_reader.py new file mode 100644 index 00000000..2c3c7794 --- /dev/null +++ b/pixie/vm2/pxic_reader.py @@ -0,0 +1,251 @@ +import pixie.vm2.interpreter as ast +import pixie.vm2.code as code +from pixie.vm2.keyword import keyword +from pixie.vm2.symbol import symbol +from pixie.vm2.primitives import true, false, nil +from pixie.vm2.array import Array +from pixie.vm2.string import char_cache +import pixie.vm2.rt as rt + +class Reader(object): + def __init__(self, filename): + self._file = open(filename, "rb") + self._cache = [] + + def __del__(self): + self._file.close() + + def read(self): + try: + return ord(self._file.read(1)[0]) + except IndexError: + raise EOFError + + def get_cache_idx(self): + idx = len(self._cache) + self._cache.append(None) + + return idx + + def set_cache(self, idx, obj): + self._cache[idx] = obj + + def get_cache(self, idx): + return self._cache[idx] + + + +bytecodes = ["CACHED_OBJECT", + "INT", + "FLOAT", + "INT_STRING", + "STRING", + "TRUE", + "FALSE", + "NIL", + "VAR", + "KEYWORD", + "SYMBOL", + "NEW_CACHED_OBJECT", + "DO", + "INVOKE", + "VAR", + "CONST", + "FN", + "LOOKUP", + "IF", + "LET", + "META", + "LINE_META", + "VAR_CONST", + "CHAR", + "VECTOR", + "RECUR"] + +for idx, x in enumerate(bytecodes): + globals()[x] = idx + + +def read_utf8_char(os): + ch = os.read() + + if ch <= 0x7F: + n = ch + bytes = 1 + elif (ch & 0xE0) == 0xC0: + n = ch & 31 + bytes = 2 + elif (ch & 0xF0) == 0xE0: + n = ch & 15 + bytes = 3 + elif (ch & 0xF8) == 0xF0: + n = ch & 7 + bytes = 4 + else: + raise AssertionError("Bad unicode character " + str(ch)) + + i = bytes - 1 + while i > 0: + + i -= 1 + n = (n << 6) | (os.read() & 0x3F) + + return unichr(n) + +def read_raw_int(os): + return os.read() | (os.read() << 8) | (os.read() << 16) | (os.read() << 24) + + + +def read_raw_string(os): + buf = [] + for x in range(read_raw_int(os)): + buf.append(read_utf8_char(os)) + + return u"".join(buf) + +def read_raw_list(os): + vals = [None] * read_raw_int(os) + for x in range(len(vals)): + vals[x] = read_object(os) + return vals + +def read_object(os): + tag = os.read() + + if tag == DO: + statements = [None] * read_raw_int(os) + for x in range(len(statements)): + statements[x] = read_object(os) + + meta = read_object(os) + return ast.Do(statements, meta) + + elif tag == INVOKE: + args = [None] * read_raw_int(os) + for x in range(len(args)): + args[x] = read_object(os) + + meta = read_object(os) + return ast.Invoke(args, meta=meta) + + elif tag == RECUR: + args = [None] * read_raw_int(os) + for x in range(len(args)): + args[x] = read_object(os) + + meta = read_object(os) + return ast.Recur(args, meta=meta) + + elif tag == NEW_CACHED_OBJECT: + idx = os.get_cache_idx() + o = read_object(os) + os.set_cache(idx, o) + return o + + elif tag == VAR: + ns = read_raw_string(os) + var_name = read_object(os) + meta = read_object(os) + return ast.VDeref(ns, var_name, meta) + + + elif tag == VAR_CONST: + ns = read_raw_string(os) + name = read_object(os) + meta = read_object(os) + return ast.VarConst(ns, name, meta) + + elif tag == CONST: + return ast.Const(read_object(os)) + + elif tag == CACHED_OBJECT: + return os.get_cache(read_raw_int(os)) + + elif tag == FN: + name_str = read_raw_string(os) + name = keyword(name_str) + args = read_raw_list(os) + closed_overs = read_raw_list(os) + body = read_object(os) + meta = read_object(os) + return ast.Fn(name=name, args=args, body=body, closed_overs=closed_overs, meta=meta) + + elif tag == LOOKUP: + return ast.Lookup(read_object(os), read_object(os)) + + elif tag == KEYWORD: + return keyword(read_raw_string(os)) + + elif tag == SYMBOL: + return symbol(read_raw_string(os)) + + elif tag == FALSE: + return false + + elif tag == TRUE: + return true + + elif tag == NIL: + return nil + + elif tag == INT: + return rt.wrap(read_raw_int(os)) + + elif tag == INT_STRING: + s = read_raw_string(os) + return rt.wrap(int(s)) + + elif tag == IF: + return ast.If(read_object(os), + read_object(os), + read_object(os), + read_object(os)) + + elif tag == LET: + bc = read_raw_int(os) + names = [None] * bc + values = [None] * bc + for idx in range(bc): + names[idx] = read_object(os) + values[idx] = read_object(os) + + body = read_object(os) + meta = read_object(os) + return ast.Let(names=names, bindings=values, body=body, meta=meta) + + elif tag == META: + line = read_object(os) + assert isinstance(line, ast.Meta) + col = read_raw_int(os) + return ast.Meta(line._c_line_tuple, col) + + elif tag == LINE_META: + file = read_raw_string(os) + line = read_raw_string(os) + line_number = read_raw_int(os) + return ast.Meta((line, file, line_number), 0) + + elif tag == STRING: + str = read_raw_string(os) + return rt.wrap(str) + + elif tag == CHAR: + str = read_raw_string(os) + return char_cache.intern(ord(str[0])) + + elif tag == VECTOR: + cnt = read_raw_int(os) + lst = [None] * cnt + for x in range(cnt): + lst[x] = read_object(os) + + return Array(lst) + + raise AssertionError("No valid handler for TAG " + bytecodes[tag]) + + +def read_file(filename): + return read_object(Reader(filename)) + + diff --git a/pixie/vm2/rt.py b/pixie/vm2/rt.py new file mode 100644 index 00000000..932fcae2 --- /dev/null +++ b/pixie/vm2/rt.py @@ -0,0 +1,171 @@ +__config__ = None +py_list = list +py_str = str +from rpython.rlib.objectmodel import specialize, we_are_translated + + +def init(): + import pixie.vm2.code as code + from pixie.vm2.object import affirm, _type_registry + from rpython.rlib.rarithmetic import r_uint, base_int + from rpython.rlib.rbigint import rbigint + from pixie.vm2.primitives import nil, true, false + from pixie.vm2.string import String + from pixie.vm2.object import Object + + + + def unwrap(fn): + if isinstance(fn, code.Var) and fn.is_defined() and hasattr(fn.deref(), "_returns"): + tp = fn.deref()._returns + if tp is bool: + def wrapper(*args): + ret = fn.invoke(py_list(args)) + if ret is nil or ret is false: + return False + return True + return wrapper + elif tp is r_uint: + return lambda *args: fn.invoke(py_list(args)).r_uint_val() + elif tp is unicode: + def wrapper(*args): + ret = fn.invoke(py_list(args)) + if ret is nil: + return None + + if not isinstance(ret, String): + from pixie.vm2.object import runtime_error + runtime_error(u"Invalid return value, expected String") + return ret._str + return wrapper + else: + assert False, "Don't know how to convert" + str(tp) + return lambda *args: fn.invoke(py_list(args)) + + if "__inited__" in globals(): + return + + import sys + #sys.setrecursionlimit(10000) # Yeah we blow the stack sometimes, we promise it's not a bug + + import pixie.vm2.code as code + import pixie.vm2.numbers as numbers + import pixie.vm2.files + import pixie.vm2.stdlib + import pixie.vm2.array + import pixie.vm2.arraymap + import pixie.vm2.custom_types + import pixie.vm2.jit_tables + import pixie.vm2.ffi + import pixie.vm2.platform + + class FakeSpace(object): + def wrap(self, x): + if isinstance(x, Object): + return x + elif isinstance(x, bool): + return true if x else false + elif isinstance(x, int): + return numbers.Integer(x) + elif isinstance(x, rbigint): + return numbers.BigInteger(x) + elif isinstance(x, float): + return numbers.Float(x) + elif isinstance(x, unicode): + return String(x) + elif isinstance(x, py_str): + return String(unicode(x)) + + elif isinstance(x, r_uint): + from pixie.vm2.numbers import SizeT + return SizeT(x) + + + elif x is None: + return nil + else: + from pixie.vm2.object import runtime_error + return runtime_error(u"Bad Wrap") + wrap._annspecialcase_ = 'specialize:argtype(1)' + + space = FakeSpace() + @specialize.argtype(0) + def wrap(x): + return space.wrap(x) + + globals()["wrap"] = wrap + + def int_val(x): + affirm(isinstance(x, numbers.Number), u"Expected number") + return x.int_val() + + globals()["int_val"] = int_val + + #f = open("pixie/stdlib.pxi") + #data = f.read() + #f.close() + #rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") + #result = nil + # + # @wrap_fn + # def run_load_stdlib(): + # with compiler.with_ns(u"pixie.stdlib"): + # while True: + # form = reader.read(rdr, False) + # if form is reader.eof: + # return result + # result = compiler.compile(form).invoke([]) + # reinit() + # + # stacklet.with_stacklets(run_load_stdlib) + + init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file", u"compile-file", + u"load-ns", u"hashmap", u"cons", u"-assoc", u"-val-at"] + for x in init_fns: + globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) + + init_vars = [u"load-paths"] + for x in init_vars: + globals()[py_str(code.munge(x))] = code.intern_var(u"pixie.stdlib", x) + + #globals()[py_str(code.munge(u"ns"))] = NS_VAR + + globals()["__inited__"] = True + + globals()["is_true"] = lambda x: False if x is false or x is nil or x is None else True + + _type_registry.set_registry(code.ns_registry) + + numbers.init() + code.init() + + import pixie.vm2.bits + + + +def unwrap_string(x): + from pixie.vm2.string import String + from pixie.vm2.object import affirm + from pixie.vm2.primitives import nil + + if x is None or x is nil: + return None + + affirm(isinstance(x, String), u"Expected String") + assert isinstance(x, String) + + return x._str + +def unwrap_keyword(x): + from pixie.vm2.keyword import Keyword + from pixie.vm2.object import affirm + + affirm(isinstance(x, Keyword), u"Expected Keyword") + assert isinstance(x, Keyword) + + return x._str + + + + + diff --git a/pixie/vm2/stdlib.py b/pixie/vm2/stdlib.py new file mode 100644 index 00000000..b28feb12 --- /dev/null +++ b/pixie/vm2/stdlib.py @@ -0,0 +1,373 @@ +import pixie.vm2.rt as rt +from pixie.vm2.code import as_var, Var, list_copy, NativeFn, extend_var +from pixie.vm2.numbers import SizeT +from pixie.vm2.object import affirm, Type, runtime_error +from pixie.vm2.primitives import nil, true, false +import pixie.vm2.code as code +import pixie.vm2.array as array +from rpython.rlib.rarithmetic import r_uint +import pixie.vm2.interpreter as interpreter +import rpython.rlib.jit as jit + +@as_var("set-var-root!") +def _set_var_root(v, r): + assert isinstance(v, Var) + v.set_root(r) + return v + +@as_var("satisfy") +def satisfy(protocol, tp): + affirm(isinstance(protocol, code.Protocol), u"First argument must be a protocol") + affirm(isinstance(tp, Type), u"Second argument must be a type") + protocol.add_satisfies(tp) + return protocol + +@as_var("pixie.stdlib.internal", "-defprotocol") +def _defprotocol(name, methods): + from pixie.vm2.string import String + from pixie.vm2.symbol import Symbol + affirm(isinstance(name, String), u"protocol name must be a symbol") + affirm(isinstance(methods, array.Array), u"protocol methods must be an array of symbols") + assert isinstance(methods, array.Array) + method_list = [] + for method_sym in methods._list: + affirm(isinstance(method_sym, String), u"protocol methods must be a vector of symbols") + assert isinstance(method_sym, String) + method_list.append(method_sym._str) + + assert isinstance(name, String) + proto = code.Protocol(name._str) + name_sym = Symbol(name._str) + code.intern_var(name_sym.get_ns(), name_sym.get_name()).set_root(proto) + for method in method_list: + method = unicode(method) + poly = code.PolymorphicFn(method, proto) + code.intern_var(name_sym.get_ns(), method).set_root(poly) + + return name + + +@as_var("polymorphic-fn") +def polymorphic_fn(name, protocol): + from pixie.vm2.string import String + affirm(isinstance(name, String), u"polymorphic functions must have string names") + affirm(isinstance(protocol, code.Protocol), u"must be a protocol") + assert isinstance(name, String) + return code.PolymorphicFn(name._str, protocol) + + +@as_var("protocol") +def protocol(name): + from pixie.vm2.string import String + affirm(isinstance(name, String), u"Protocol names must be strings") + assert isinstance(name, String) + return code.Protocol(name._str) + +@as_var("extend") +def _extend(proto_fn, tp, fn): + if isinstance(proto_fn, interpreter.EffectFunction): + proto_fn = proto_fn._inner_fn + + if not isinstance(proto_fn, code.PolymorphicFn): + runtime_error(u"Fist argument to extend should be a PolymorphicFn not a " + proto_fn.type().name()) + + affirm(isinstance(tp, Type) or isinstance(tp, code.Protocol), u"Second argument to extend must be a Type or Protocol") + proto_fn.extend(tp, fn) + return nil + +@as_var("variadic-fn") +def _variadic_fn(required_arity, fn): + arity = required_arity.int_val() + return code.VariadicCode(required_arity=arity, code=fn) + +@as_var("-effect-fn") +def _variadic_fn(inner_fn): + + return interpreter.EffectFunction(inner_fn) + +as_var("-with-handler")(interpreter.WithHandler()) + +@as_var("multi-arity-fn") +def _multi_arity_fn__args(args): + from pixie.vm2.string import String + nm = args[0] + affirm(isinstance(nm, String), u"Function name must be string") + assert isinstance(nm, String) + arities = {} + + required_arity = 0 + rest_fn = None + + idx = 1 + while idx + 1 < len(args): + arity = args[idx].int_val() + if arity < 0: + required_arity = -arity + rest_fn = args[idx + 1] + else: + arities[arity] = args[idx + 1] + idx += 2 + + return code.MultiArityFn(nm._str, arities, required_arity, rest_fn) + +class Apply(NativeFn): + @jit.unroll_safe + def invoke_k(self, args, stack): + from pixie.vm2.array import Array + last_itm = args[len(args) - 1] + affirm(isinstance(last_itm, Array), u"Final argument in -apply must be an array") + assert isinstance(last_itm, Array) + fn = args[0] + argc = r_uint(len(args) - 2) + out_args = [None] * (argc + len(last_itm._list)) + + list_copy(args, 1, out_args, 0, argc) + + for x in range(len(last_itm._list)): + out_args[argc + x] = last_itm._list[x] + + return fn.invoke_k(out_args, stack) + +as_var("-apply")(Apply()) + + +@as_var("-satisfies?") +def _satisfies(proto, o): + affirm(isinstance(proto, code.Protocol), u"proto must be a Protocol") + + return true if proto.satisfies(o.type()) else false + + +@as_var("-instance?") +def _instance(c, o): + from pixie.vm2.object import istypeinstance + affirm(isinstance(c, Type), u"c must be a type") + + return true if istypeinstance(o.type(), c) else false + + +@as_var("-internal-get-field") +def _get_field(inst, k): + return inst.get_field(k, nil) + +@as_var("identical?") +def identical(a, b): + return true if a is b else false + +@as_var("-internal-to-str") +def _internal_to_str(x): + return rt.wrap(x.to_str()) + +@as_var("-internal-to-repr") +def _internal_to_repr(x): + return rt.wrap(x.to_repr()) + +@as_var("-internal-get-ns") +def _internal_get_ns(x): + return rt.wrap(x.get_ns()) + +@as_var("-internal-get-name") +def _internal_get_name(x): + return rt.wrap(x.get_name()) + +@as_var("-internal-get-hash") +def _internal_get_name(x): + return rt.wrap(x.get_hash()) + + +@as_var("-internal-store-hash") +def _internal_store_hash(x, h): + x.store_hash(h.r_uint_val()) + return nil + +@as_var("-internal-identity-hash") +def _internal_identity_hash(x): + from rpython.rlib.objectmodel import compute_identity_hash + return rt.wrap(compute_identity_hash(x)) + + +@as_var("-internal-int") +def _internal_int(x): + return rt.wrap(x.int_val()) + +@as_var("-blocking-println") +def _blocking_println(x): + print rt.unwrap_string(x) + return x + + +@as_var("-string-builder") +def _string_builder(): + from pixie.vm2.string_builder import StringBuilder + return StringBuilder() + +@as_var("-add-to-string-builder") +def _add_to_string_builder(sb, x): + from pixie.vm2.string import String, Character + if isinstance(x, String): + sb.add_str(x._str) + return sb + elif isinstance(x, Character): + sb.add_str(unichr(x._char_val)) + else: + runtime_error(u"Expected string or char", u"pixie.stdlib.IllegalArgumentException") + +@as_var("-finish-string-builder") +def _finish_string_builder(sb): + return rt.wrap(sb.to_str()) + +@as_var("size-t") +def size_t(i): + return SizeT(i.r_uint_val()) + + +@as_var("type") +def type(x): + return x.type() + + +@as_var("the-ns") +def the_ns(ns_name): + affirm(ns_name.get_ns() is None, u"the-ns takes a un-namespaced symbol") + + return code.ns_registry.get(ns_name.get_name(), nil) + + + + +### NS Helpers + +@as_var("-add-refer") +def refer_syms(in_ns_nm, other_ns_nm, as_nm): + from pixie.vm2.keyword import Keyword + from pixie.vm2.code import ns_registry + + in_ns = ns_registry.get(in_ns_nm.get_name(), None) + affirm(in_ns is not None, u"Can't locate namespace " + in_ns_nm.get_name()) + other_ns = ns_registry.get(other_ns_nm.get_name(), None) + affirm(other_ns is not None, u"Can't locate namespace " + in_ns_nm.get_name()) + + in_ns.add_refer(other_ns, as_nm.get_name()) + +@as_var("-refer-all") +def refer_all(in_ns_nm, other_ns_sym): + from pixie.vm2.keyword import Keyword + from pixie.vm2.code import ns_registry + + in_ns = ns_registry.get(in_ns_nm.get_name(), None) + affirm(in_ns is not None, u"Can't locate namespace " + in_ns_nm.get_name()) + + in_ns.get_refer(other_ns_sym.get_name()).refer_all() + + +@as_var("-refer-var") +def refer_all(in_ns_nm, other_ns_sym, old, new): + from pixie.vm2.keyword import Keyword + from pixie.vm2.code import ns_registry + + in_ns = ns_registry.get(in_ns_nm.get_name(), None) + affirm(in_ns is not None, u"Can't locate namespace " + in_ns_nm.get_name()) + + in_ns.get_refer(other_ns_sym.get_name()).add_var_alias(old.get_name(), new.get_name()) + + +@as_var("-in-ns") +def in_ns(ns_name): + ns = code.ns_registry.find_or_make(ns_name.get_name()) + ns.include_stdlib() + + return nil + +@as_var("-run-external-extends") +def run_external_extends(): + for var, tp, f in code.init_ctx: + var.deref().extend(tp, f) + + +@as_var("set-dynamic!") +def set_dynamic(var): + affirm(isinstance(var, Var), u"set-dynamic! expects a var as an argument") + var.set_dynamic() + return var + + +@as_var("resolve-in") +def _var(ns, nm): + if ns is nil: + return nil + if not isinstance(ns, code.Namespace): + ns = code.ns_registry.find_or_make(ns.get_name()) + + var = ns.resolve_in_ns_ex(nm.get_ns(), nm.get_name()) + return var if var is not None else nil + + +class PartialFunction(code.NativeFn): + _immutable_fields_ = ["_partial_f", "_partial_args"] + def __init__(self, f, args): + code.NativeFn.__init__(self) + self._partial_f = f + self._partial_args = args + + @jit.unroll_safe + def invoke_k(self, args, stack): + new_args = [None] * (len(args) + len(self._partial_args)) + + for x in range(len(self._partial_args)): + new_args[x] = self._partial_args[x] + + plen = len(self._partial_args) + + for x in range(len(args)): + new_args[plen + x] = args[x] + + + return self._partial_f.invoke_k(new_args, stack) + + +@as_var("partial") +@jit.unroll_safe +def _partial__args(args): + """(partial f & args) + Creates a function that is a partial application of f. Thus ((partial + 1) 2) == 3""" + + f = args[0] + + new_args = [None] * (len(args) - 1) + + for x in range(len(new_args)): + new_args[x] = args[x + 1] + + return PartialFunction(f, new_args) + +from pixie.vm2.code import BaseCode + +@as_var("set-macro!") +def set_macro(f): + affirm(isinstance(f, BaseCode), u"Only code objects can be macros") + f.set_macro() + return f + +@as_var("macro?") +def macro_QMARK_(f): + return true if isinstance(f, BaseCode) and f.is_macro() else false + + +@extend_var("pixie.stdlib", "-deref", Var) +def __deref(self): + assert isinstance(self, Var) + return self.deref() + + +@extend_var("pixie.stdlib", "-name", Var) +def __deref(self): + assert isinstance(self, Var) + return rt.wrap(self._name) + + +@extend_var("pixie.stdlib", "-namespace", Var) +def __deref(self): + assert isinstance(self, Var) + return rt.wrap(self._ns) + + diff --git a/pixie/vm2/string.py b/pixie/vm2/string.py new file mode 100644 index 00000000..586af663 --- /dev/null +++ b/pixie/vm2/string.py @@ -0,0 +1,250 @@ +import pixie.vm2.rt as rt +from pixie.vm2.object import Object, Type, affirm, runtime_error +from pixie.vm2.code import extend_var, as_var, wrap_fn +from pixie.vm2.primitives import nil, true, false +from pixie.vm2.numbers import Integer +#import pixie.vm2.stdlib as proto +#import pixie.vm2.util as util +from rpython.rlib.rarithmetic import intmask, r_uint +import rpython.rlib.jit as jit +#from pixie.vm2.libs.pxic.util import add_marshall_handlers + +class String(Object): + _type = Type(u"pixie.stdlib.String") + + def type(self): + return String._type + + def __init__(self, s): + #assert isinstance(s, unicode) + self._str = s + + + def get_name(self): + return self._str + + def get_ns(self): + return None + + +@as_var("-str-len") +def str_len(self): + assert isinstance(self, String) + return rt.wrap(len(self._str)) + +@as_var("-str-nth") +def str_len(self, idx): + assert isinstance(self, String) + i = idx.int_val() + return char_cache.intern(ord(self._str[i])) + +@as_var("pixie.string", "-substring") +def substring3(a, start, end): + affirm(isinstance(a, String), u"First argument must be a string") + affirm(isinstance(start, Integer) and isinstance(end, Integer), u"Second and third argument must be integers") + start = start.int_val() + end = end.int_val() + if start >= 0 and end >= 0: + return rt.wrap(a.get_name()[start:end]) + else: + runtime_error(u"Second and third argument must be non-negative integers") + + +# +# +# @extend(proto._str, String) +# def _str(x): +# return x +# +# @extend(proto._repr, String) +# def _repr(self): +# res = u"" +# assert isinstance(self, String) +# for c in self._str: +# if c == "\"": +# res += u"\\\"" +# elif c == "\n": +# res += u"\\n" +# elif c == "\t": +# res += u"\\t" +# elif c == "\b": +# res += u"\\b" +# elif c == "\f": +# res += u"\\f" +# elif c == "\r": +# res += u"\\r" +# else: +# res += c +# return rt.wrap(u"\"" + res + u"\"") +# +# @extend(proto._count, String) +# def _count(self): +# assert isinstance(self, String) +# return rt.wrap(len(self._str)) +# +# @extend(proto._nth, String) +# def _nth(self, idx): +# assert isinstance(self, String) +# i = idx.int_val() +# if 0 <= i < len(self._str): +# return Character(ord(self._str[i])) +# affirm(False, u"Index out of Range") +# +# @extend(proto._nth_not_found, String) +# def _nth_not_found(self, idx, not_found): +# assert isinstance(self, String) +# i = idx.int_val() +# if 0 <= i < len(self._str): +# return Character(ord(self._str[i])) +# return not_found +# + +@extend_var(u"pixie.stdlib", u"-eq", String) +def _eq(self, v): + assert isinstance(self, String) + if not isinstance(v, String): + return false + return true if self._str == v._str else false +# +class Character(Object): + _type = Type(u"pixie.stdlib.Character") + _immutable_fields_ = ["_char_val"] + + def type(self): + return Character._type + + def __init__(self, i): + assert isinstance(i, int) + self._char_val = i + + def char_val(self): + return self._char_val + + def int_val(self): + return self._char_val + + def to_str(self): + assert isinstance(self, Character) + return u"" + unichr(self.char_val()) + + def to_repr(self): + assert isinstance(self, Character) + cv = self.char_val() + if cv < 128: + return u"\\"+unicode(chr(cv)) + return u"FIXME" + #hexv = rt.name(rt.bit_str(rt.wrap(self.char_val()), rt.wrap(4))) + #return rt.wrap(u"\\u" + u"0" * (4 - len(hexv)) + hexv) + + +class CharCache(object): + def __init__(self): + self._char_cache = {} + self._rev = 0 + + @jit.elidable_promote() + def intern_inner(self, ival, rev): + return self._char_cache.get(ival, None) + + def intern(self, ival): + v = self.intern_inner(ival, self._rev) + if not v: + v = Character(ival) + self._char_cache[ival] = v + self._rev += 1 + + return v + + +char_cache = CharCache() + +@as_var("char") +def char(val): + affirm(isinstance(val, Integer), u"First argument must be an Integer") + return char_cache.intern(val.int_val()) + + +# @wrap_fn +# def write_char(obj): +# assert isinstance(obj, Character) +# return rt.wrap(obj._char_val) +# +# @wrap_fn +# def read_char(obj): +# return Character(obj.int_val()) +# +# add_marshall_handlers(Character._type, write_char, read_char) +# +# @extend(proto._str, Character) +# def _str(self): +# assert isinstance(self, Character) +# return rt.wrap(u"" + unichr(self.char_val())) +# +# @extend(proto._repr, Character) +# def _repr(self): +# assert isinstance(self, Character) +# cv = self.char_val() +# if cv < 128: +# return rt.wrap(u"\\"+unicode(chr(cv))) +# hexv = rt.name(rt.bit_str(rt.wrap(self.char_val()), rt.wrap(4))) +# return rt.wrap(u"\\u" + u"0" * (4 - len(hexv)) + hexv) +# +# @extend(proto._eq, Character) +# def _eq(self, obj): +# assert isinstance(self, Character) +# if self is obj: +# return true +# if not isinstance(obj, Character): +# return false +# return true if self.char_val() == obj.char_val() else false +# +# @extend(proto._hash, Character) +# def _hash(self): +# return rt.wrap(intmask(util.hash_int(r_uint(self.char_val())))) +# +# @as_var("char") +# def char(val): +# affirm(isinstance(val, Integer), u"First argument must be an Integer") +# return Character(val.int_val()) +# +# @extend(_add, Character._type, Integer._type) +# def _add(a, b): +# assert isinstance(a, Character) and isinstance(b, Integer) +# return rt._add(rt.wrap(a.char_val()), b) +# +# @extend(_add, Character._type, Character._type) +# def _add(a, b): +# assert isinstance(a, Character) and isinstance(b, Character) +# return Character(a.char_val() + b.char_val()) +# +# +# @extend(proto._name, String) +# def _name(self): +# return self +# +# @extend(proto._namespace, String) +# def _namespace(self): +# return nil +# +# @extend(proto._hash, String) +# def _hash(self): +# assert isinstance(self, String) +# return rt.wrap(intmask(util.hash_unencoded_chars(self._str))) + + +@as_var("pixie.string", "starts-with?") +def startswith(a, b): + return rt.wrap(a.get_name().startswith(b.get_name())) + + +@as_var("pixie.string", "ends-with?") +def endswith(a, b): + return rt.wrap(a.get_name().endswith(b.get_name())) + + +@as_var("pixie.string", "replace") +def str_replace(s, sfrom, sto): + affirm(isinstance(s, String), u"Expected string") + affirm(isinstance(sfrom, String), u"Expected string") + affirm(isinstance(sto, String), u"Expected string") + return rt.wrap(s.get_name().replace(sfrom.get_name()[0], sto.get_name()[0])) \ No newline at end of file diff --git a/pixie/vm2/string_builder.py b/pixie/vm2/string_builder.py new file mode 100644 index 00000000..299b283b --- /dev/null +++ b/pixie/vm2/string_builder.py @@ -0,0 +1,21 @@ +import pixie.vm2.rt as rt +from pixie.vm2.object import Object, Type +from pixie.vm2.code import as_var, extend +import pixie.vm2.stdlib as proto + +class StringBuilder(Object): + _type = Type(u"pixie.stdlib.StringBuilder") + + def type(self): + return StringBuilder._type + + def __init__(self): + self._strs = [] + + def add_str(self, s): + self._strs.append(s) + return self + + def to_str(self): + return u"".join(self._strs) + diff --git a/pixie/vm2/symbol.py b/pixie/vm2/symbol.py new file mode 100644 index 00000000..876b6aac --- /dev/null +++ b/pixie/vm2/symbol.py @@ -0,0 +1,113 @@ +import pixie.vm2.object as object +from pixie.vm2.primitives import nil, true, false +from pixie.vm2.code import extend_var, as_var +from pixie.vm2.string import String +import pixie.vm2.rt as rt +#import pixie.vm2.util as util +from rpython.rlib.rarithmetic import intmask, r_uint + + +class Symbol(object.Object): + _type = object.Type(u"pixie.stdlib.Symbol") + def __init__(self, s, meta=nil): + #assert isinstance(s, unicode) + self._str = s + self._w_name = None + self._w_ns = None + self._hash = r_uint(0) + self._meta = meta + + def type(self): + return Symbol._type + + def init_names(self): + if self._w_name is None: + s = self._str.split(u"/") + if len(s) == 2: + self._w_ns = rt.wrap(s[0]) + self._w_name = rt.wrap(s[1]) + elif len(s) == 1: + self._w_name = rt.wrap(s[0]) + self._w_ns = nil + else: + self._w_ns = rt.wrap(s[0]) + self._w_name = rt.wrap(u"/".join(s[1:])) + + def with_meta(self, meta): + return Symbol(self._str, meta) + + def meta(self): + return self._meta + + def get_name(self): + self.init_names() + return rt.unwrap_string(self._w_name) + + def get_ns(self): + self.init_names() + return rt.unwrap_string(self._w_ns) + + + def to_str(self): + return self._str + + def to_repr(self): + return self._str + + def store_hash(self, hash): + self._hash = hash + + def get_hash(self): + return self._hash + +def symbol(s): + return Symbol(s) +# +@extend_var("pixie.stdlib", "-eq", Symbol) +def _eq(self, other): + assert isinstance(self, Symbol) + if not isinstance(other, Symbol): + return false + return true if self._str == other._str else false + + +# @extend(proto._str, Symbol) +# def _str(self): +# assert isinstance(self, Symbol) +# return rt.wrap(self._str) +# +# @extend(proto._name, Symbol) +# def _name(self): +# assert isinstance(self, Symbol) +# self.init_names() +# return self._w_name +# +# @extend(proto._namespace, Symbol) +# def _namespace(self): +# assert isinstance(self, Symbol) +# self.init_names() +# return self._w_ns +# +# @extend(proto._hash, Symbol) +# def _hash(self): +# assert isinstance(self, Symbol) +# if self._hash == 0: +# self._hash = util.hash_unencoded_chars(self._str) +# return rt.wrap(intmask(self._hash)) +# +@as_var("symbol") +def _symbol(s): + if not isinstance(s, String): + from pixie.vm2.object import runtime_error + runtime_error(u"Symbol name must be a string") + return symbol(s.get_name()) + +@extend_var("pixie.stdlib", "-meta", Symbol) +def _meta(self): + assert isinstance(self, Symbol) + return self.meta() + +@extend_var("pixie.stdlib", "-with-meta", Symbol) +def _with_meta(self, meta): + assert isinstance(self, Symbol) + return self.with_meta(meta) diff --git a/target.py b/target.py index 66f397c5..b34154fc 100644 --- a/target.py +++ b/target.py @@ -91,7 +91,7 @@ def inner_invoke(self, args): if newline_pos > 0: data = data[newline_pos:] - rt.load_reader(StringReader(unicode_from_utf8(data))) + rt.load_reader(MetaDataReader(StringReader(unicode_from_utf8(data)), unicode(self._file))) except WrappedException as ex: print "Error: ", ex._ex.__repr__() os._exit(1) diff --git a/target2.py b/target2.py new file mode 100644 index 00000000..e24e4d21 --- /dev/null +++ b/target2.py @@ -0,0 +1,151 @@ + +import pixie.vm2.interpreter as i +from pixie.vm2.interpreter import run_stack +from pixie.vm2.object import StackCell +import pixie.vm2.rt as rt +from pixie.vm2.primitives import nil, true, false +import pixie.vm2.code as code +from pixie.vm2.keyword import keyword as kw +from pixie.vm2.symbol import symbol as sym +from pixie.vm2.numbers import parse_number +from pixie.vm2.pxic_reader import read_file, read_object, Reader +from rpython.rlib.objectmodel import we_are_translated + +rt.init() +import sys + +def testit(max): + rdr = Reader("./bootstrap.pxic") + while True: + try: + obj = read_object(rdr) + except EOFError: + break + if not we_are_translated(): + print ".", + sys.stdout.flush() + run_stack(None, i.InterpretK(obj, None)) + + return None + #pixie_code = read_file("/tmp/bootstrap.pxic") + #return run_stack(None, i.InterpretK(pixie_code, None)) + +#val = testit() +#print val.int_val(), val + +def entry_point(args): + #s = rt.wrap(u"Foo") + from pixie.vm2.string import String + v = parse_number(u"1") + + s = String(u"Foo") + max = 10000 #int(args[1]) + + val = testit(max) + return 43 + +def entry_point0(): + #s = rt.wrap(u"Foo") + from pixie.vm2.string import String + v = parse_number(u"1") + + s = String(u"Foo") + max = 10000 #int(args[1]) + + val = testit(max) + return 43 + +## JIT STUFF + + +from rpython.jit.codewriter.policy import JitPolicy +from rpython.rlib.jit import JitHookInterface, Counters +from rpython.rlib.rfile import create_stdio +from rpython.annotator.policy import AnnotatorPolicy +from rpython.rtyper.lltypesystem import lltype +from rpython.jit.metainterp import warmspot + +def run_child(glob, loc): + interp = loc['interp'] + graph = loc['graph'] + interp.malloc_check = False + + def returns_null(T, *args, **kwds): + return lltype.nullptr(T) + interp.heap.malloc_nonmovable = returns_null # XXX + + from rpython.jit.backend.llgraph.runner import LLGraphCPU + #LLtypeCPU.supports_floats = False # for now + apply_jit(interp, graph, LLGraphCPU) + + +def apply_jit(interp, graph, CPUClass): + print 'warmspot.jittify_and_run() started...' + policy = Policy() + warmspot.jittify_and_run(interp, graph, [], policy=policy, + listops=True, CPUClass=CPUClass, + backendopt=True, inline=True) + +def run_debug(argv): + from rpython.rtyper.test.test_llinterp import get_interpreter + + # first annotate and rtype + try: + interp, graph = get_interpreter(entry_point0, [], backendopt=False, + #config=config, + #type_system=config.translation.type_system, + policy=Policy()) + except Exception, e: + print '%s: %s' % (e.__class__, e) + pdb.post_mortem(sys.exc_info()[2]) + raise + + # parent process loop: spawn a child, wait for the child to finish, + # print a message, and restart + #unixcheckpoint.restartable_point(auto='run') + + from rpython.jit.codewriter.codewriter import CodeWriter + CodeWriter.debug = True + run_child(globals(), locals()) + +#stacklet.global_state = stacklet.GlobalState() + +class DebugIFace(JitHookInterface): + def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations): + print "Aborted Trace, reason: ", Counters.counter_names[reason], logops, greenkey_repr + #from rpython.rlib.objectmodel import we_are_translated + #import pdb; pdb.set_trace() + #exit(0) + pass + + def before_compile_bridge(self, debug_info): + print "Compiling Bridge", debug_info + pass + +import sys, pdb + +class Policy(JitPolicy, AnnotatorPolicy): + def __init__(self): + JitPolicy.__init__(self, DebugIFace()) + +def jitpolicy(driver): + return JitPolicy(jithookiface=DebugIFace()) + +def target(*args): + import pixie.vm.rt as rt + driver = args[0] + driver.exe_name = "pixie-vm2" + rt.__config__ = args[0].config + + + print "ARG INFO: ", args + + + return entry_point, None + +import rpython.config.translationoption +print rpython.config.translationoption.get_combined_translation_config() + +if __name__ == "__main__": + #run_debug(sys.argv) + entry_point([])