Merge pull request #27 from cceckman/slongfield/start_parse

Create a better start-line parsing module.

Author: cceckman

http_server/parse_start.py

@@ -0,0 +1,146 @@
+from amaranth import Module
+from amaranth.lib.wiring import In, Out, Component, connect, flipped
+from amaranth.lib import stream
+
+from string_contains_match import StringContainsMatch
+from stream_demux import StreamDemux
+import stream_utils
+
+class ParseStart(Component):
+    """
+    Parser for the start-line of an HTTP request.
+
+    Parameters
+    ----------
+    paths: list[str]
+           Valid paths to match
+
+    Attributes
+    ----------
+    input:    Stream(8), in
+              Data stream to match
+    reset:    Signal(1), in
+              Reset and await new input
+    done:     Signal(1), out
+              Indicates that the "\r\n" end-of-line sequence was seen.
+    method:   list(Signal(1)), out
+              Bitfield of matched methods. The 0th field indicates no match.
+              METHOD_* constants can be used for decode.
+    path:     list(Signal(1)), out
+              Bitfield of matched paths. The 0th field indicates no match.
+              Other matches are in the order from the paths parameter.
+    protocol: list(Siganl(1)), out
+              Bitfield of matched protocol. The 0th field indicates no match.
+              PROTOCOL_* constants can be used for decode.
+    """
+
+    METHOD_NO_MATCH = 0
+    METHOD_GET = 1
+    METHOD_POST = 2
+
+    PROTOCOL_NO_MATCH = 0
+    PROTOCOL_HTTP1_0 = 1
+
+    def __init__(self, paths):
+        super().__init__({
+            "input": In(stream.Signature(8)),
+            "reset": In(1),
+            "done": Out(1),
+            "method": Out(3),
+            "path": Out(len(paths)+1),
+            "protocol": Out(2),
+        })
+        self._paths = paths
+
+    def elaborate(self, _platform):
+        m = Module()
+
+        resets = []
+
+        # TODO: #4 - Evaluate using StringMatch instead of StringContainsMatch. Here and below.
+        #            Check https://en.wikipedia.org/wiki/HTTP_request_smuggling cases.
+        method_stream = stream.Signature(8).create()
+        get_matcher = m.submodules.get_matcher = StringContainsMatch("GET")
+        resets.append(get_matcher.reset)
+        m.d.comb += self.method[self.METHOD_GET].eq(get_matcher.accepted)
+
+        post_matcher = m.submodules.post_matcher = StringContainsMatch("POST")
+        resets.append(post_matcher.reset)
+        m.d.comb += self.method[self.METHOD_POST].eq(post_matcher.accepted)
+
+        any_method_match = stream_utils.tree_or(m, [get_matcher.accepted, post_matcher.accepted])
+        m.d.comb += self.method[0].eq(~any_method_match)
+
+        stream_utils.fanout_stream(m, method_stream, [get_matcher.input, post_matcher.input])
+
+        path_stream = stream.Signature(8).create()
+        path_streams = []
+        path_match = []
+        for i,path in enumerate(self._paths):
+            matcher = m.submodules[f"path_matchers_{i}"] = StringContainsMatch(path)
+            path_streams.append(matcher.input)
+            resets.append(matcher.reset)
+            m.d.comb += self.path[i+1].eq(matcher.accepted)
+            path_match.append(matcher.accepted)
+        stream_utils.fanout_stream(m, path_stream, path_streams)
+        any_path_match = stream_utils.tree_or(m, path_match)
+        m.d.comb += self.path[0].eq(~any_path_match)
+
+        # TODO: #4 - If we want to get out of the stone age, should match more than HTTP/1.0
+        #            That being said, silicon is kind of like a stone, right?
+        protocol_match = m.submodules.protocol_match = StringContainsMatch("HTTP/1.0")
+        m.d.comb += self.protocol[self.PROTOCOL_NO_MATCH].eq(~protocol_match.accepted) 
+        m.d.comb += self.protocol[self.PROTOCOL_HTTP1_0].eq(protocol_match.accepted) 
+
+        with m.FSM():
+            with m.State("reset"):
+                for r in resets:
+                    m.d.sync += r.eq(1)
+                m.d.sync += self.done.eq(0)
+                m.next = "match_method"
+            with m.State("match_method"):
+                m.next = "match_method"
+                for r in resets:
+                    m.d.sync += r.eq(0)
+                m.d.comb += [
+                    method_stream.valid.eq(self.input.valid),
+                    method_stream.payload.eq(self.input.payload),
+                    self.input.ready.eq(method_stream.ready),
+                ]
+                with m.If(self.input.valid & (self.input.payload == ord(' '))):
+                    m.d.comb += self.input.ready.eq(1)
+                    m.next = "match_path"
+            with m.State("match_path"):
+                m.next = "match_path"
+                m.d.comb += [
+                    path_stream.valid.eq(self.input.valid),
+                    path_stream.payload.eq(self.input.payload),
+                    self.input.ready.eq(path_stream.ready),
+                ]
+                with m.If(self.input.valid & (self.input.payload == ord(' '))):
+                    m.d.comb += self.input.ready.eq(1)
+                    m.next = "match_protocol"
+            with m.State("match_protocol"):
+                m.next = "match_protocol"
+                # connect results in warning about combinatorial signals
+                m.d.comb += [
+                    protocol_match.input.valid.eq(self.input.valid),
+                    protocol_match.input.payload.eq(self.input.payload),
+                    self.input.ready.eq(protocol_match.input.ready),
+                ]
+                with m.If(self.input.valid & (self.input.payload == ord('\r'))):
+                    m.d.comb += self.input.ready.eq(1)
+                    m.next = "match_end"
+            with m.State("match_end"):
+                m.d.comb += self.input.ready.eq(1)
+                m.next = "match_end"
+                # TODO: #4 - Should error if this isn't \n, and setup to return a 
+                # HTTP 400 Bad Request error.
+                with m.If(self.input.valid & (self.input.payload == ord('\n'))):
+                    m.next = "done"
+            with m.State("done"):
+                m.next = "done"
+                m.d.sync += self.done.eq(1)
+
+        return m
+

http_server/parse_start_test.py

@@ -0,0 +1,110 @@
+import sys
+from amaranth.sim import Simulator
+
+from parse_start import ParseStart
+from stream_fixtures import StreamSender
+
+paths = ["/led", "/count"]
+PATH_NO_MATCH = 0
+PATH_LED = 1
+PATH_COUNT = 2
+
+def run_test(send_line, check):
+    dut = ParseStart(["/led", "/count"])
+    sender = StreamSender(stream=dut.input)
+    sim = Simulator(dut)
+    sim.add_clock(1e-6)
+    checked = False
+
+    async def driver(ctx):
+        nonlocal checked
+        ctx.set(dut.reset, 1)
+        ctx.tick()
+        ctx.set(dut.reset, 0)
+
+        while not sender.done:
+            await ctx.tick()
+        check(ctx, dut)
+        checked = True
+
+    sim.add_testbench(driver)
+    sim.add_process(sender.send_passive(map(ord,send_line)))
+
+    with sim.write_vcd(sys.stdout):
+        sim.run_until(0.0001)
+    assert checked
+
+def test_good_parse():
+    def check(ctx, dut):
+        assert ctx.get(dut.method[dut.METHOD_NO_MATCH])==0
+        assert ctx.get(dut.method[dut.METHOD_GET])==0
+        assert ctx.get(dut.method[dut.METHOD_POST])==1
+        assert ctx.get(dut.path[PATH_NO_MATCH])==0
+        assert ctx.get(dut.path[PATH_LED])==1
+        assert ctx.get(dut.path[PATH_COUNT])==0
+        assert ctx.get(dut.protocol[dut.PROTOCOL_NO_MATCH])==0
+        assert ctx.get(dut.protocol[dut.PROTOCOL_HTTP1_0])==1
+
+    run_test("POST /led HTTP/1.0\r\n", check)
+
+
+def test_no_method_match():
+    def check(ctx, dut):
+        assert ctx.get(dut.method[dut.METHOD_NO_MATCH])==1
+        assert ctx.get(dut.method[dut.METHOD_GET])==0
+        assert ctx.get(dut.method[dut.METHOD_POST])==0
+        assert ctx.get(dut.path[PATH_NO_MATCH])==0
+        assert ctx.get(dut.path[PATH_LED])==1
+        assert ctx.get(dut.path[PATH_COUNT])==0
+        assert ctx.get(dut.protocol[dut.PROTOCOL_NO_MATCH])==0
+        assert ctx.get(dut.protocol[dut.PROTOCOL_HTTP1_0])==1
+
+    run_test("REQUEST /led HTTP/1.0\r\n", check)
+
+def test_no_protocol_match():
+    def check(ctx, dut):
+        assert ctx.get(dut.method[dut.METHOD_NO_MATCH])==0
+        assert ctx.get(dut.method[dut.METHOD_GET])==1
+        assert ctx.get(dut.method[dut.METHOD_POST])==0
+        assert ctx.get(dut.path[PATH_NO_MATCH])==0
+        assert ctx.get(dut.path[PATH_LED])==0
+        assert ctx.get(dut.path[PATH_COUNT])==1
+        assert ctx.get(dut.protocol[dut.PROTOCOL_NO_MATCH])==1
+        assert ctx.get(dut.protocol[dut.PROTOCOL_HTTP1_0])==0
+
+    run_test("GET /count HTTP/3.0\r\n", check)
+
+
+def test_no_path_match():
+    def check(ctx, dut):
+        assert ctx.get(dut.method[dut.METHOD_NO_MATCH])==0
+        assert ctx.get(dut.method[dut.METHOD_GET])==1
+        assert ctx.get(dut.method[dut.METHOD_POST])==0
+        assert ctx.get(dut.path[PATH_NO_MATCH])==1
+        assert ctx.get(dut.path[PATH_LED])==0
+        assert ctx.get(dut.path[PATH_COUNT])==0
+        assert ctx.get(dut.protocol[dut.PROTOCOL_NO_MATCH])==0
+        assert ctx.get(dut.protocol[dut.PROTOCOL_HTTP1_0])==1
+
+    run_test("GET /index.html HTTP/1.0\r\n", check)
+
+def test_double_start_line():
+    def check(ctx, dut):
+        assert ctx.get(dut.method[dut.METHOD_NO_MATCH])==0
+        assert ctx.get(dut.method[dut.METHOD_GET])==1
+        assert ctx.get(dut.method[dut.METHOD_POST])==0
+        assert ctx.get(dut.path[PATH_NO_MATCH])==1
+        assert ctx.get(dut.path[PATH_LED])==0
+        assert ctx.get(dut.path[PATH_COUNT])==0
+        assert ctx.get(dut.protocol[dut.PROTOCOL_NO_MATCH])==0
+        assert ctx.get(dut.protocol[dut.PROTOCOL_HTTP1_0])==1
+
+    # TODO: #4 - This should raise an error so the HTTP responder can return a 400 Bad Request.
+    run_test("GET /index.html HTTP/1.0\rPOST /help HTTP/1.1\r\n", check)
+
+if __name__ == "__main__":
+    test_good_parse()
+    test_no_method_match()
+    test_no_path_match()
+    test_no_protocol_match()
+    test_double_start_line()

http_server/simple_led_http.py

@@ -2,6 +2,7 @@
 from amaranth.lib.wiring import In, Out, Component, connect
 
 from printer import Printer
+from parse_start import ParseStart
 from stream_mux import StreamMux
 from stream_demux import StreamDemux
 from string_match import StringMatch
@@ -43,14 +44,11 @@ def elaborate(self, _platform):
         parser_demux = m.submodules.parser_demux = StreamDemux(mux_width=4, stream_width=8)
         connect(m, self.session.inbound.data, parser_demux.input)
 
-        # Match the header for an HTTP/1.0 request to the LED path.
-        # TODO: If we want to match more than one path, could probably have some common
-        #       matching for the method and protocol. Also, if we want to get out of the
-        #       stone age, this could be HTTP/1.1.
-        led_start = "POST /led HTTP/1.0\r\n"
-        led_start_matcher = m.submodules.led_start_matcher = StringMatch(led_start)
-        HTTP_PARSER_LED = 0
-        connect(m, led_start_matcher.input, parser_demux.outs[HTTP_PARSER_LED])
+        # TODO: #4 - Add packet count and RFC2324 endpoints
+        MATCHED_LED_PATH = 1 # start_matcher path match is in the order the paths are connected.
+        start_matcher = m.submodules.start_matcher = ParseStart(["/led"])
+        HTTP_PARSER_START = 0
+        connect(m, start_matcher.input, parser_demux.outs[HTTP_PARSER_START])
 
         HTTP_PARSER_HEADERS = 1
         skip_headers = m.submodules.end_of_header_matcher = StringContainsMatch("\r\n\r\n")
@@ -70,7 +68,7 @@ def elaborate(self, _platform):
         m.d.comb += parser_demux.outs[HTTP_PARSER_SINK].ready.eq(1)
 
         ## Responders
-        response_mux = m.submodules.response_mux = StreamMux(mux_width=2, stream_width=8)
+        response_mux = m.submodules.response_mux = StreamMux(mux_width=3, stream_width=8)
         connect(m, response_mux.out, self.session.outbound.data)
 
         ok_response = "\r\n".join(
@@ -107,46 +105,67 @@ def elaborate(self, _platform):
                 not_found_printer.en.eq(1),
         ]
 
+        not_allowed_response = "\r\n".join(
+                ["HTTP/1.0 405 Method Not Allowed",
+                    "Host: Fomu",
+                    "Content-Type: text/plain; charset=utf-8",
+                    "",
+                    "",
+                    '🛑']) + "\r\n"
+        not_allowed_response = not_allowed_response.encode("utf-8")
+        not_allowed_printer = m.submodules.not_allowed_printer = Printer(not_allowed_response)
+        RESPONSE_405 = 2
+        connect(m, not_allowed_printer.output, response_mux.input[RESPONSE_405])
+        send_405 = [
+                response_mux.select.eq(RESPONSE_405),
+                parser_demux.select.eq(HTTP_PARSER_SINK),
+                not_allowed_printer.en.eq(1),
+        ]
+
+
         with m.FSM():
             with m.State("reset"):
                 m.d.comb += [
-                        led_start_matcher.reset.eq(1),
+                        start_matcher.reset.eq(1),
                         skip_headers.reset.eq(1),
                         led_body_handler.reset.eq(1),
                 ]
                 m.next = "idle"
             with m.State("idle"):
-                m.d.comb += led_start_matcher.reset.eq(0)
-                m.d.sync += parser_demux.select.eq(HTTP_PARSER_LED)
+                m.d.comb += start_matcher.reset.eq(0)
+                m.d.sync += parser_demux.select.eq(HTTP_PARSER_START)
                 m.d.sync += response_mux.select.eq(RESPONSE_OK)
                 m.next = "idle"
                 with m.If(self.session.inbound.active):
                     m.next = "parsing_start"
                     m.d.sync += self.session.outbound.active.eq(1)
             with m.State("parsing_start"):
                 m.next = "parsing_start"
-                # Input finished before header matched, or header failed to match
-                with m.If(~self.session.inbound.active | led_start_matcher.rejected):
-                    m.next = "writing"
-                    m.d.sync += send_404
                 # start line matched successfully
-                with m.Elif(led_start_matcher.accepted):
-
+                with m.If(start_matcher.done):
                     m.next = "parsing_header"
                     m.d.sync += parser_demux.select.eq(HTTP_PARSER_HEADERS)
             with m.State("parsing_header"):
                 m.next = "parsing_header"
                 with m.If(skip_headers.accepted):
-                    m.next = "parsing_body"
-                    # TODO: #4 - Should pick the body parser based on the method+path from start
-                    m.d.sync += parser_demux.select.eq(HTTP_PARSER_LED_BODY)
+                    with m.If(start_matcher.method[start_matcher.METHOD_POST] & 
+                              start_matcher.path[MATCHED_LED_PATH]):
+                        m.next = "parsing_led_body"
+                        m.d.sync += parser_demux.select.eq(HTTP_PARSER_LED_BODY)
+                    with m.Elif(start_matcher.method[start_matcher.METHOD_GET] & 
+                              start_matcher.path[MATCHED_LED_PATH]):
+                        m.next = "writing"
+                        m.d.sync += send_405
+                    with m.Else():
+                        m.next = "writing"
+                        m.d.sync += send_404
                 with m.Elif(~self.session.inbound.active):
                     m.next = "writing"
                     # TODO: #4 - Should send a different error code besides 404 if the
                     #            headers fail to parse before end-of-session.
                     m.d.sync += send_404
-            with m.State("parsing_body"): # TODO: #4 - Make the specific body depend on the path
-                m.next = "parsing_body"
+            with m.State("parsing_led_body"): # TODO: #4 - Make body parsing state more generic.
+                m.next = "parsing_led_body"
                 with m.If(led_body_handler.accepted):
                     m.next = "writing"
                     m.d.sync += send_ok
@@ -160,10 +179,12 @@ def elaborate(self, _platform):
                 m.d.sync += [
                         ok_printer.en.eq(0),
                         not_found_printer.en.eq(0),
+                        not_allowed_printer.en.eq(0),
                         self.session.outbound.active.eq(1),
                 ]
                 with m.If(  ((response_mux.select == RESPONSE_OK) & ok_printer.done)
-                          | ((response_mux.select == RESPONSE_404) & not_found_printer.done)):
+                          | ((response_mux.select == RESPONSE_404) & not_found_printer.done)
+                          | ((response_mux.select == RESPONSE_405) & not_allowed_printer.done)):
                     m.d.sync += self.session.outbound.active.eq(0)
                     # Can finish writing before all the input is collected,
                     # since a bad request migh trigger an early 404. Wait