Started tokenizing, still struggling with strings

datatypes.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from enum import auto, Enum
 from typing import Any
+import re
 
 
 class Datatype(Enum):
@@ -40,11 +41,15 @@ class Variable:
 
 NONE_TYPE = Variable(None, Datatype.NONE, None)
 
+STR_MATCH_BEGIN = re.compile(r"^\"")
+STR_MATCH_END = re.compile(r"\"$")
+VAR_ASSIGN_TOKEN = re.compile(r"[A-Za-z]+ ?= ?")
+
 VAR_MATCH = {
-    r"null": Datatype.NULL,
+    re.compile(r"null"): Datatype.NULL,
-    r"\/\/.+": Datatype.COMMENT,
+    re.compile(r"\/\/.+"): Datatype.COMMENT,
-    r"\".+\"": Datatype.STRING,
+    re.compile(r"\".+\""): Datatype.STRING,
-    r"-?\d+\.?\d{0,}?\+\d+\.?\d{0,}?i": Datatype.COMPLEX,
+    re.compile(r"-?\d+\.?\d{0,}?\+\d+\.?\d{0,}?i"): Datatype.COMPLEX,
-    r"-?\d+\.\d{0,}": Datatype.FLOAT,
+    re.compile(r"-?\d+\.\d{0,}"): Datatype.FLOAT,
-    r"-?\d+": Datatype.INTEGER,
+    re.compile(r"-?\d+"): Datatype.INTEGER,
 }

pyne.py

@@ -4,45 +4,107 @@ from decorators import deprecated, todo, wip
 from enum import Enum
 from pyne_errors import ParserError, FileError
 import datatypes
+import os
 import re
 import sys
 
 
+def debugprint(output):
+    if "PYNE_DEBUG" in os.environ:
+        print(output)
+
+
 @dataclass
 class PyneObject:
     value: datatypes.Variable
 
 
+VAR_STORE: list[PyneObject] = []
+
+
 class Pyne:
     def __init__(self, path: str, args: list[str]):
         self.args = args
         self.path = path if self._validate(path) else None
 
-    @todo(
+    @wip
-        "Tokenize the parser so it doesn't try to match the whole line, gotta find a way for it to do the strings though"
-    )
     def parse(self) -> list[PyneObject]:
-        objects: list[PyneObject] = []
+        # VAR_STORE: list[PyneObject] = []
+        col = 1
+        is_str_begin = False
         with open(self.path) as f:
             for row, line in enumerate(f.readlines(), 1):
+                varname = ""
+                strbuf = ""
+                debugprint(line)
+                if "=" in line:
+                    debugprint("if '=' in line")
+                    varname = line.split("=")[0].split(" ")[0]
+                    debugprint(f"{varname = }")
+                    col += len(varname) + line.find("=")
+                splitline = (
+                    line[line.find("=") + 1 if varname != "" else 0 :]
+                    .strip()
+                    .split(" ")
+                )
+                debugprint(f"{splitline = }")
+                for token in splitline:
+                    debugprint(f"{token = }")
+                    if is_str_begin:
+                        debugprint("is_str_begin = true")
+                        strbuf += token
+                    if datatypes.STR_MATCH_BEGIN.match(token):
+                        debugprint("Matched opening quote")
+                        is_str_begin = True
+                        strbuf += token
+                    if datatypes.STR_MATCH_END.match(token):
+                        debugprint("Matched end quote")
+                        is_str_begin = False
+                        VAR_STORE.append(
+                            PyneObject(
+                                datatypes.Variable(varname, datatypes.STRING, strbuf)
+                            )
+                        )
                     for var in datatypes.VAR_MATCH:
-                    regex = re.compile(var)
+                        if token == varname:
-                    res = regex.match(line[:-1])
+                            break
+                        if token == "":
+                            break
+                        if is_str_begin:
+                            break
+                        res = var.match(token)
                         if res:
-                        objects.append(
+                            VAR_STORE.append(
                                 PyneObject(
                                     datatypes.Variable(
-                                    None, datatypes.VAR_MATCH[var], line[:-1]
+                                        varname, datatypes.VAR_MATCH[var], token
                                     )
                                 )
                             )
                             break
-                    continue
                     else:
-                    raise ParserError(f"[{self.path}:{row}] Token {line} not found.")
+                        raise ParserError(
-        if len(objects) == 0:
+                            f"{self.path}:{row}:{col}: Invalid token `{token}`"
-            objects.append(datatypes.NONE_TYPE)
+                        )
-        return objects
+                col += len(token)
+                debugprint(f"{VAR_STORE=}")
+                # for var in datatypes.VAR_MATCH:
+                #     res = var.match(line[:-1])
+                #     if res:
+                #         VAR_STORE.append(
+                #             PyneObject(
+                #                 datatypes.Variable(
+                #                     None, datatypes.VAR_MATCH[var], line[:-1]
+                #                 )
+                #             )
+                #         )
+                #         break
+                #     continue
+                # else:
+                #     raise ParserError(
+                #         f"[{self.path}:{row}] Token `{line[:-1]}` not found."
+                #     )
+        return VAR_STORE
 
     def _validate(self, path: str) -> bool:
         """
@@ -59,13 +121,23 @@ def main() -> int:
     parser.add_argument("file", nargs=1, help="The input file")
     args = parser.parse_args()
     print(args)
+    # obj = PyneObject(
+    #     datatypes.Variable(
+    #         "test",
+    #         datatypes.Datatype.STRING,
+    #         datatypes.Data(datatypes.Datatype.STRING, '"Hello World"'),
+    #     )
+    # )
+    # # dt = datatypes.Data(datatypes.Datatype.INTEGER, 24)
+    # # obj.value.value = dt
+    # dt = datatypes.Data(datatypes.Datatype.STRING, '"Goodbye World"')
+    # obj.value.value = dt
+    # print(obj.value.value.type, obj.value.value.value)
+    # return 0
     pyne = Pyne(args.file[0], None)
-    try:
     for obj in pyne.parse():
         print(obj)
     return 0
-    except ParserError as e:
-        print(f"Found invalid token: {e}")
 
 
 if __name__ == "__main__":