diff --git a/http-server.pxi b/http-server.pxi new file mode 100644 index 00000000..229002a5 --- /dev/null +++ b/http-server.pxi @@ -0,0 +1,30 @@ +(ns http-server + (:require [pixie.io.tcp :as tcp] + [pixie.streams.utf8 :as utf8] + [pixie.io :refer [buffered-output-stream flush]] + [pixie.parser :refer [input-stream-cursor]] + [pixie.parser.http :as http])) + +(defn send-response [conn data] + (using [buffered (buffered-output-stream conn) + utf-stream (utf8/utf8-output-stream buffered)] + (utf8/write-string utf-stream (:protocol data)) + (utf8/write-char utf-stream \space) + (utf8/write-string utf-stream (str (:status data))) + (utf8/write-string utf-stream "\r\n\r\n") + (utf8/write-string utf-stream (:body data)))) + +(defn print-headers [conn] + (comment + (doseq [s (slurp conn)] + (println s (int s)))) + + (println ((:REQUEST http/HTTP-REQUEST) (input-stream-cursor conn))) + (send-response conn {:protocol "HTTP/1.0" + :status "200 OK" + :body "IT WORKS!!!!"}) + (dispose! conn) + (println "FINISHED")) + +(let [server (tcp/tcp-server "0.0.0.0" 4242 print-headers)] + (println "Started")) diff --git a/pixie/io.pxi b/pixie/io.pxi index 9929018e..28bbe98e 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -37,20 +37,21 @@ IDisposable (-dispose! [this] (dispose! uvbuf) - (fs_close fp)) - IReduce - (-reduce [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc)))))) - + (fs_close fp))) + +(defn -reduce-input-stream-fn [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc))))) + +(extend -reduce IInputStream -reduce-input-stream-fn) (defn open-read {:doc "Open a file for reading, returning a IInputStream" @@ -162,7 +163,7 @@ 0 (uv/uv_buf_t))) -(defn spit +(defn spit "Writes the content to output. Output must be a file or an IOutputStream." [output content] (cond @@ -173,17 +174,17 @@ buffered-output-stream utf8/utf8-output-stream-rf) (str content)) - + (satisfies? IOutputStream output) (transduce (map identity) (-> output buffered-output-stream utf8/utf8-output-stream-rf) (str content)) - + :else (throw "Expected a string or IOutputStream"))) -(defn slurp +(defn slurp "Reads in the contents of input. Input must be a filename or an IInputStream" [input] (let [stream (cond diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index 2ca440f4..ff98626d 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -29,6 +29,7 @@ ;(dispose! uv-buf) (uv/uv_read_stop stream) (st/run-and-process k (or + (when (= nread uv/UV_EOF) 0) (st/exception-on-uv-error nread) nread)) (catch ex diff --git a/pixie/parser.pxi b/pixie/parser.pxi index e1b12a26..5ee610de 100644 --- a/pixie/parser.pxi +++ b/pixie/parser.pxi @@ -1,5 +1,8 @@ (ns pixie.parser - (:require [pixie.stdlib :as s])) + (:require [pixie.stdlib :as s] + [pixie.streams :refer [read IInputStream IInputStream]])) + +(def DEFAULT-BUFFER-SIZE 1024) ;; This file contans a small framework for writing generic parsers in Pixie. Although the generated @@ -22,8 +25,9 @@ (next! [this] (set-field! this :idx (inc idx))) (current [this] - (when (< idx (count s)) - (nth s idx))) + (if (< idx (count s)) + (nth s idx) + ::at-end)) (snapshot [this] idx) (rewind! [this val] @@ -52,11 +56,13 @@ the cursor to the next element." [pred] (fn [cursor] - (if (pred (current cursor)) - (let [value (current cursor)] - (next! cursor) - value) - fail))) + (let [cur (current cursor)] + (if (and + (pred cur) + (not (identical? cur ::at-end))) + (do (next! cursor) + cur) + fail)))) (defprotocol IParserGenerator @@ -269,14 +275,15 @@ (defn eat "Eagerly parses as many values as possible until g fails. Discards the result, returns nil." [g] - (fn [cursor] - (loop [] - (let [prev-pos (snapshot cursor) - v (g cursor)] - (if (identical? v fail) - (do (rewind! cursor prev-pos) - nil) - (recur)))))) + (let [g (to-parser g)] + (fn [cursor] + (loop [] + (let [prev-pos (snapshot cursor) + v (g cursor)] + (if (identical? v fail) + (do (rewind! cursor prev-pos) + nil) + (recur))))))) (defn maybe "Always succeeds, returns nil when the input did not match the parser." @@ -310,3 +317,40 @@ (def digits (parse-if (set "1234567890"))) (def whitespace (parse-if #{\newline \return \space \tab})) + + + + +;; Represents an input stream that implements ICursor + +(defn -input-stream-buffer-seq + "Creates a lazy seq of buffers retreived from the input stream" + [is] + (let [buf (gc-buffer DEFAULT-BUFFER-SIZE) + nread (read is buf DEFAULT-BUFFER-SIZE)] + (when (pos? nread) + (cons buf (lazy-seq (-input-stream-buffer-seq is)))))) + +(defrecord InputStreamSnapshot [buffer-cell idx]) + +(deftype InputStreamCursor [buffer-cell idx] + ICursor + (next! [this] + (if (= (inc idx) (count (first buffer-cell))) + (do (set-field! this :buffer-cell (next buffer-cell)) + (set-field! this :idx 0)) + (set-field! this :idx (inc idx)))) + (current [this] + (let [val (when (s/and buffer (< idx (count (first buffer-cell)))) + (char (nth (first buffer-cell) idx)))] + val)) + (at-end? [this] (= nil buffer-cell)) + (snapshot [this] + (->InputStreamSnapshot buffer-cell idx)) + (rewind! [this snapshot] + (assert (instance? InputStreamSnapshot snapshot) (str "Must provide an input stream snapshot ")) + (set-field! this :buffer-cell (:buffer-cell snapshot)) + (set-field! this :idx (:idx snapshot)))) + +(defn input-stream-cursor [is] + (->InputStreamCursor (-input-stream-buffer-seq is) 0)) diff --git a/pixie/parser/http.pxi b/pixie/parser/http.pxi new file mode 100644 index 00000000..1e388107 --- /dev/null +++ b/pixie/parser/http.pxi @@ -0,0 +1,49 @@ +(ns pixie.parser.http + (:require [pixie.parser :refer :all])) + + +(defn merge-rf + ([] {}) + ([acc] acc) + ([acc itm] + (merge acc itm))) + +(defparser HTTP-REQUEST [] + NEWLINE (and (eat \space) + (maybe \return) + \newline) + NOT-NEWLINE (parse-if (complement #{\newline \return})) + TO-END-OF-LINE (one+chars NOT-NEWLINE) + TO-COLON (one+chars (parse-if (complement #{\newline \return \:}))) + + WHITESPACE? (eat \space) + + NOT-WHITESPACE (parse-if (complement #{\space \newline \return})) + + REQUEST-LINE (and (one+chars NOT-WHITESPACE) -> method + WHITESPACE? + (one+chars NOT-WHITESPACE) -> path + WHITESPACE? + (one+chars NOT-WHITESPACE) -> protocol + NEWLINE + <- {:method method + :path path + :protocol protocol}) + + HEADER (and TO-COLON -> k + \: + WHITESPACE? + TO-END-OF-LINE -> v + NEWLINE + <- {k v}) + + REQUEST (and REQUEST-LINE -> request + (zero+ HEADER merge-rf) -> headers +; \return +; \newline + <- (assoc request :headers headers))) + +(def test-request +"GET /foo/bar http/1.1\r\nHost: foo.bar.com\r\n\r\n") + +( (:REQUEST HTTP-REQUEST) (string-cursor test-request)) diff --git a/pixie/parser/json.pxi b/pixie/parser/json.pxi index 321d71c1..2d0abbab 100644 --- a/pixie/parser/json.pxi +++ b/pixie/parser/json.pxi @@ -109,3 +109,7 @@ (if (failure? result) (println (current c) (snapshot c)) result))) + +(defn read-one [c] + (assert (satisfies? ICursor c)) + ((:ENTRY JSONParser) c)) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 30e0ada3..0051c662 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -27,7 +27,8 @@ (defn exception-on-uv-error [result] (when (neg? result) - (->ThrowException (str "UV Error: " (uv/uv_err_name result))))) + (->ThrowException [(keyword (str ":pixie.io/" (uv/uv_err_name result))) + (str "UV Error: " (uv/uv_err_name result))]))) (defn call-cc [f] diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d8783a10..f4f6b005 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -905,11 +905,8 @@ If further arguments are passed, invokes the method named by symbol, passing the but returns the opposite truth value"} [f] (assert (fn? f) "Complement must be passed a function") - (fn - ([] (not (f))) - ([x] (not (f x))) - ([x y] (not (f x y))) - ([x y & more] (not (apply f x y more))))) + (fn [& args] + (not (apply f args)))) (defn constantly [x] {:doc "Return a function that always returns x, no matter what it is called with." @@ -1266,7 +1263,8 @@ and implements IAssociative, ILookup and IObject." `(= (. self ~field) (. other ~field))) fields))) `(-hash [self] - (throw [:pixie.stdlib/NotImplementedException "not implemented"]))] + (hash ~fields))] + deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] `(do ~type-from-map ~deftype-decl))) @@ -2229,7 +2227,7 @@ If the number of arguments is even and no clause matches, throws an exception." result# (do ~@body)] ~@(map (fn [nm] `(-dispose! ~nm)) - names) + (reverse names)) result#))) (defn pst diff --git a/pixie/streams/utf8.pxi b/pixie/streams/utf8.pxi index bc9b4044..c30a96c2 100644 --- a/pixie/streams/utf8.pxi +++ b/pixie/streams/utf8.pxi @@ -24,8 +24,7 @@ (write-byte out (bit-or 0x80 (bit-and ch 0x3F)))) :else (assert false (str "Cannot encode a UTF8 character of code " ch))))) IDisposable - (-dispose! [this] - (dispose! out))) + (-dispose! [this])) (deftype UTF8InputStream [in bad-char] @@ -52,8 +51,7 @@ (throw (str "Invalid UTF8 character decoded: " n))) (char n)))))) IDisposable - (-dispose! [this] - (dispose! in))) + (-dispose! [this])) (defn utf8-input-stream "Creates a UTF8 decoder that reads characters from the given IByteInputStream. If a bad character is found @@ -76,3 +74,11 @@ (assert (char? chr)) (write-char fp chr) nil)))) + +(defn write-string [os s] + (reduce + (fn [os c] + (write-char os c) + os) + os + s)) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index b8b92512..5bd0e77d 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -217,15 +217,16 @@ class Buffer(object.Object): def type(self): return Buffer._type - def __init__(self, size): + def __init__(self, size, auto_free): self._size = size self._used_size = 0 + self._auto_free = auto_free self._buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor="raw") def __del__(self): - #lltype.free(self._buffer, flavor="raw") - pass + if self._auto_free: + lltype.free(self._buffer, flavor="raw") def set_used_size(self, size): self._used_size = size @@ -248,8 +249,7 @@ def capacity(self): def free_data(self): lltype.free(self._buffer, flavor="raw") - - + self._auto_free = False @extend(proto._dispose_BANG_, Buffer) def _dispose_voidp(self): @@ -271,7 +271,11 @@ def _count(self): @as_var("buffer") def buffer(size): - return Buffer(size.int_val()) + return Buffer(size.int_val(), False) + +@as_var("gc-buffer") +def gc_buffer(size): + return Buffer(size.int_val(), True) @as_var("buffer-capacity") def buffer_capacity(buffer): diff --git a/tests/pixie/tests/parser/test-json.pxi b/tests/pixie/tests/parser/test-json.pxi index e79ef6cd..eff5a578 100644 --- a/tests/pixie/tests/parser/test-json.pxi +++ b/tests/pixie/tests/parser/test-json.pxi @@ -1,5 +1,7 @@ (ns pixie.tests.parser.test-json (:require [pixie.test :refer :all] + [pixie.io :refer [slurp open-read]] + [pixie.parser :refer [failure? input-stream-cursor]] [pixie.parser.json :as json])) @@ -31,3 +33,16 @@ "{\"foo\": 42}" {"foo", 42} "{\"foo\": 42, \"bar\":null}" {"foo" 42 "bar" nil})) + +(deftest test-streaming-json-file + (let [filename "tests/pixie/tests/parser/test-json-data.json" + cursor (-> filename + open-read + input-stream-cursor) + streamed-result (json/read-one cursor) + slurped-result (-> "tests/pixie/tests/parser/test-json-data.json" + slurp + json/read-string)] + (assert (not (failure? streamed-result))) + (assert (not (failure? slurped-result))) + (assert= slurped-result streamed-result)))