diff --git a/README.tlm b/README.tlm index 4551ca2..0a19cb3 100644 --- a/README.tlm +++ b/README.tlm @@ -1,6 +1,5 @@ ! TypoML - !! Installation TypoML requires the Odin compiler. Build from source: @@ -19,6 +18,8 @@ TypoML reads from stdin and outputs HTML to stdout: cat input.tlm | ./TypoML > output.html ``` +--- + !! Syntax !!! Text Formatting @@ -31,6 +32,8 @@ cat input.tlm | ./TypoML > output.html `_` - H_2_O_4_U - Subscript `^` - Δ = b^2^ - 4ac - Superscript +--- + !!! Headings \! Heading 1 @@ -38,6 +41,14 @@ cat input.tlm | ./TypoML > output.html \!!! Heading 3 (And so on) +Result: + +! Heading 1 +!! Heading 2 +!!! Heading 3 + +--- + !!! Links \#[Link text](url) - Standard link @@ -45,6 +56,15 @@ cat input.tlm | ./TypoML > output.html \#[url] - Auto-link (no alt text) [Link text](\#heading) - Header link (anchor) (can also be done like an auto-link, then the text is auto-generated from the heading) +Result: + +#[Link text](https://example.com) - Standard link +#(https://example.com)[Link text] - Can go either way +#[https://example.com] - Auto-link (no alt text) +[Link text](#headings) - Header link (anchor) + +--- + !!! Alignment Placed at the end of start of a line @@ -53,6 +73,14 @@ Placed at the end of start of a line \<< - Left aligned \^^ - Center aligned +Result: + +This text is right aligned >> +This text is left aligned << +This text is center aligned ^^ + +--- + !!! Embedded Content \$[url] - Auto-detect type @@ -68,6 +96,13 @@ Types: - `iframe`, `frame`, `f` - `script`, `s`, `javascript`, `js` +Result: + +$[https://upload.wikimedia.org/wikipedia/commons/b/b2/Voynich_manuscript_recipe_example_107r_crop.jpg|img] +$[staircase.mp3] + +--- + !!! Lists - Item 1 @@ -80,6 +115,8 @@ Unordered lists use - or . as markers. Ordered lists use numbers followed by periods. +--- + !!! Code Blocks ```language @@ -88,6 +125,8 @@ code here Inline code uses backticks: `code` +--- + !!! Tables | Header 1 | Header 2 | Header 3 | @@ -97,11 +136,20 @@ Inline code uses backticks: `code` Do not need a header to work, just the first separator row (`|---|---|---|`) Can be aligned using the alignment markers +--- + !!! Blockquotes \> This is a blockquote \> Multiple lines are supported +Result: + +> This is a blockquote +> Multiple lines are supported + +--- + !!! Footnotes [\&footnote] - Footnote reference @@ -109,23 +157,55 @@ Can be aligned using the alignment markers Can be numbers or names +Result: + +Here is a footnote reference [&demo] + +[&&demo] And here is its definition + +--- + !!! Variables \$(name=value) - Define a variable \$(name) - Use a variable +Result: + +$(greeting=Hello, world!) + +The value of `greeting` is: $(greeting) + +Variables can be accessed in JavaScript using `tlm.` (without the `<>`) + +--- + !!! Script Blocks \$[script] JavaScript code here \$[/script] +Result: + +$[script] +console.log("Hello from TypoML!"); +$[/script] + +(Open your browser's console to see the output) + +--- + !!! Escaping Standard escaping using a backslash (`\`) !!! Horizontal Rules -\--\-, \__\_, or \==\= +\---, \___, or \=== Three or more, have to be on their own line + +Result: + +--- diff --git a/src/html.odin b/src/html.odin deleted file mode 100644 index a2f8782..0000000 --- a/src/html.odin +++ /dev/null @@ -1,468 +0,0 @@ -package main - -import "core:fmt" -import "core:strings" -import "core:unicode" - -escape_html :: proc(s: string, allocator := context.temp_allocator) -> string { - escaped := decode_escapes(s, allocator) - - b := strings.builder_make(allocator) - for ch in escaped { - switch ch { - case '&': - strings.write_string(&b, "&") - case '<': - strings.write_string(&b, "<") - case '>': - strings.write_string(&b, ">") - case '"': - strings.write_string(&b, """) - case: - strings.write_rune(&b, ch) - } - } - return strings.to_string(b) -} - -get_text_content :: proc(token: ^Token, b: ^strings.Builder) { - if token.type == .Text { - if v, ok := token.value.(string); ok { - strings.write_string(b, v) - } - } - for child in token.children { - get_text_content(child, b) - } -} - -check_only_var_defs :: proc(token: ^Token) -> bool { - for child in token.children { - if child.type == .VariableDef || child.type == .BreakLine { - continue - } - if child.type == .Text { - if child.value != nil { - if v, ok := child.value.(string); ok && v != "" { - return false - } - } - has_non_var_def := false - for gc in child.children { - if gc.type != .VariableDef { - has_non_var_def = true - break - } - } - if has_non_var_def { - return false - } - continue - } - return false - } - return true -} - -slugifyifyifyify :: proc(text: string) -> string { - b := strings.builder_make(context.temp_allocator) - - for ch in text { - lower_ch := unicode.to_lower(ch) - - if unicode.is_letter(lower_ch) || unicode.is_digit(lower_ch) { - strings.write_rune(&b, lower_ch) - } else if lower_ch == ' ' || lower_ch == '_' || lower_ch == '-' { - strings.write_rune(&b, '-') - } - } - - res := strings.to_string(b) - - for { - new_res, replaced := strings.replace_all(res, "--", "-", context.temp_allocator) - if !replaced { - break - } - res = new_res - } - - res = strings.trim(res, "-") - - return res -} - -print_html :: proc(token: ^Token) { - switch token.type { - case TokenType.Root: - has_footnotes := false - for child in token.children { - if child.type == .FootnoteDef && !has_footnotes { - fmt.print("
\n") - has_footnotes = true - } - print_html(child) - } - - case TokenType.Header: - level := 1 - if v, ok := token.value.(int); ok { - level = v - } - - b := strings.builder_make(context.temp_allocator) - for child in token.children { - get_text_content(child, &b) - } - slug := slugifyifyifyify(strings.to_string(b)) - - fmt.printf("", level, escape_html(slug)) - for child in token.children { - print_html(child) - } - fmt.printf("\n", level) - - case TokenType.Paragraph: - only_var_defs := check_only_var_defs(token) - if !only_var_defs { - fmt.print("

") - for child in token.children { - print_html(child) - } - fmt.print("

\n") - } else { - for child in token.children { - if child.type != .BreakLine { - print_html(child) - } - } - } - - case TokenType.Blockquote: - fmt.print("
\n") - for child in token.children { - print_html(child) - } - fmt.print("
\n") - - case TokenType.UnorderedList: - fmt.print("\n") - - case TokenType.OrderedList: - fmt.print("
    \n") - for child in token.children { - print_html(child) - } - fmt.print("
\n") - - case TokenType.ListItem: - fmt.print("
  • ") - for child in token.children { - print_html(child) - } - fmt.print("
  • \n") - - case TokenType.Bold: - fmt.print("") - for child in token.children { - print_html(child) - } - fmt.print("") - - case TokenType.Italics: - fmt.print("") - for child in token.children { - print_html(child) - } - fmt.print("") - - case TokenType.Strikethrough: - fmt.print("") - for child in token.children { - print_html(child) - } - fmt.print("") - - case TokenType.Table: - fmt.print("\n") - for child in token.children { - print_html(child) - } - fmt.print("
    \n") - - case TokenType.TableHeaderRow: - fmt.print("") - for child in token.children { - align := "" - if len(child.children) > 0 { - first_child := child.children[0] - if first_child.type == .Left { - align = "left" - } else if first_child.type == .Right { - align = "right" - } else if first_child.type == .Middle { - align = "center" - } - } - if align != "" { - fmt.printf("", align) - for grandchild in child.children[0].children { - print_html(grandchild) - } - } else { - fmt.print("") - for grandchild in child.children { - print_html(grandchild) - } - } - fmt.print("") - } - fmt.print("\n") - - case TokenType.TableRow: - fmt.print("") - for child in token.children { - print_html(child) - } - fmt.print("\n") - - case TokenType.TableCell: - align := "" - if len(token.children) > 0 { - first_child := token.children[0] - if first_child.type == .Left { - align = "left" - } else if first_child.type == .Right { - align = "right" - } else if first_child.type == .Middle { - align = "center" - } - } - if align != "" { - fmt.printf("", align) - for child in token.children[0].children { - print_html(child) - } - } else { - fmt.print("") - for child in token.children { - print_html(child) - } - } - fmt.print("") - - case TokenType.TableSeparator: - fmt.print("\n") - - case TokenType.Left: - fmt.print("
    ") - for child in token.children { - print_html(child) - } - fmt.print("
    ") - - case TokenType.Right: - fmt.print("
    ") - for child in token.children { - print_html(child) - } - fmt.print("
    ") - - case TokenType.Middle: - fmt.print("
    ") - for child in token.children { - print_html(child) - } - fmt.print("
    ") - - case TokenType.FootnoteRef: - if id, ok := token.value.(string); ok { - fmt.printf( - "%s", - escape_html(id), - escape_html(id), - escape_html(id), - ) - } - - case TokenType.FootnoteDef: - if id, ok := token.value.(string); ok { - fmt.printf( - "
    ^ ", - escape_html(id), - escape_html(id), - ) - for child in token.children { - print_html(child) - } - fmt.print("
    \n") - } - - case TokenType.CodeBlock: - lang := "" - if v, ok := token.value.(string); ok && v != "" { - lang = v - } - if lang != "" { - fmt.printf("
    ", lang)
    -		} else {
    -			fmt.print("
    ")
    -		}
    -		for i := 0; i < len(token.children); i += 1 {
    -			print_html(token.children[i])
    -			if i + 1 < len(token.children) {
    -				fmt.print("\n")
    -			}
    -		}
    -		fmt.print("\n
    \n") - - case TokenType.InlineCode: - if code, ok := token.value.(string); ok { - fmt.printf("%s", escape_html(code)) - } - - case TokenType.HorizontalRule: - fmt.print("
    \n") - - case TokenType.Underline: - fmt.print("") - for child in token.children { - print_html(child) - } - fmt.print("") - - case TokenType.Superscript: - fmt.print("") - for child in token.children { - print_html(child) - } - fmt.print("") - - case TokenType.Subscript: - fmt.print("") - for child in token.children { - print_html(child) - } - fmt.print("") - - case TokenType.HeaderLink: - if target, ok := token.value.(string); ok { - slug := slugifyifyifyify(target) - fmt.printf("", escape_html(slug)) - for child in token.children { - print_html(child) - } - fmt.print("") - } - - case TokenType.Link: - if url, ok := token.value.(string); ok { - fmt.printf("", escape_html(url)) - for child in token.children { - print_html(child) - } - fmt.print("") - } - - case TokenType.Image: - if url, ok := token.value.(string); ok { - b := strings.builder_make(context.temp_allocator) - for child in token.children { - get_text_content(child, &b) - } - alt_text := strings.to_string(b) - fmt.printf("\"%s\"", escape_html(url), escape_html(alt_text)) - } - - case TokenType.Audio: - if url, ok := token.value.(string); ok { - mime := detect_mime_type(url) - b := strings.builder_make(context.temp_allocator) - for child in token.children { - get_text_content(child, &b) - } - alt_text := strings.to_string(b) - if alt_text == "" { - alt_text = url - } - fmt.printf("\n", escape_html(url), escape_html(mime)) - fmt.printf("\n", escape_html(url)) - fmt.printf("\n") - fmt.printf("\n") - fmt.printf("%s\n", escape_html(url), escape_html(alt_text)) - fmt.print("\n") - } - - case TokenType.Video: - if url, ok := token.value.(string); ok { - mime := detect_mime_type(url) - b := strings.builder_make(context.temp_allocator) - for child in token.children { - get_text_content(child, &b) - } - alt_text := strings.to_string(b) - if alt_text == "" { - alt_text = url - } - fmt.printf("\n", escape_html(url), escape_html(mime)) - fmt.printf("\n", escape_html(url)) - fmt.printf("\n") - fmt.printf("\n") - fmt.printf("%s\n", escape_html(url), escape_html(alt_text)) - fmt.print("\n") - } - - case TokenType.Iframe: - if url, ok := token.value.(string); ok { - fmt.printf("", escape_html(url)) - } - - case TokenType.Script: - if url, ok := token.value.(string); ok { - fmt.printf("", escape_html(url)) - } - - case TokenType.ScriptBlock: - if content, ok := token.value.(string); ok { - fmt.printf("", content) - } - - case TokenType.Variable: - if name, ok := token.value.(string); ok { - fmt.printf("", escape_html(name)) - for child in token.children { - print_html(child) - } - fmt.print("") - } - - case TokenType.VariableDef: - if name, ok := token.value.(string); ok { - b := strings.builder_make(context.temp_allocator) - for child in token.children { - get_text_content(child, &b) - } - val := strings.to_string(b) - fmt.printf( - "", - escape_html(name), - escape_html(val), - ) - } - - case TokenType.BreakLine: - fmt.print("
    \n") - - case TokenType.Text: - if v, ok := token.value.(string); ok { - fmt.print(escape_html(v)) - } - for child in token.children { - print_html(child) - } - } -} diff --git a/src/main.odin b/src/main.odin index 6605edc..d1d9e12 100644 --- a/src/main.odin +++ b/src/main.odin @@ -1,5 +1,8 @@ package main +import "./parser" +import "./renderer" +import "./token" import "core:fmt" import "core:os" import "core:strings" @@ -10,91 +13,91 @@ read_stdin :: proc() -> []string { return strings.split_lines(content) } -print_token :: proc(token: ^Token, depth: int = 0) { +print_token :: proc(tok: ^token.Token, depth: int = 0) { indent := strings.repeat(" ", depth, context.temp_allocator) type_str := "" - switch token.type { - case TokenType.Root: + switch tok.type { + case token.TokenType.Root: type_str = "Root" - case TokenType.Text: + case token.TokenType.Text: type_str = "Text" - case TokenType.Italics: + case token.TokenType.Italics: type_str = "Italics" - case TokenType.Bold: + case token.TokenType.Bold: type_str = "Bold" - case TokenType.BreakLine: + case token.TokenType.BreakLine: type_str = "BreakLine" - case TokenType.Header: + case token.TokenType.Header: type_str = "Header" - case TokenType.Paragraph: + case token.TokenType.Paragraph: type_str = "Paragraph" - case TokenType.Blockquote: + case token.TokenType.Blockquote: type_str = "Blockquote" - case TokenType.UnorderedList: + case token.TokenType.UnorderedList: type_str = "UnorderedList" - case TokenType.OrderedList: + case token.TokenType.OrderedList: type_str = "OrderedList" - case TokenType.ListItem: + case token.TokenType.ListItem: type_str = "ListItem" - case TokenType.Strikethrough: + case token.TokenType.Strikethrough: type_str = "Strikethrough" - case TokenType.Table: + case token.TokenType.Table: type_str = "Table" - case TokenType.TableRow: + case token.TokenType.TableRow: type_str = "TableRow" - case TokenType.TableHeaderRow: + case token.TokenType.TableHeaderRow: type_str = "TableHeaderRow" - case TokenType.TableCell: + case token.TokenType.TableCell: type_str = "TableCell" - case TokenType.TableSeparator: + case token.TokenType.TableSeparator: type_str = "TableSeparator" - case TokenType.Left: + case token.TokenType.Left: type_str = "Left" - case TokenType.Right: + case token.TokenType.Right: type_str = "Right" - case TokenType.Middle: + case token.TokenType.Middle: type_str = "Middle" - case TokenType.FootnoteDef: + case token.TokenType.FootnoteDef: type_str = "FootnoteDef" - case TokenType.FootnoteRef: + case token.TokenType.FootnoteRef: type_str = "FootnoteRef" - case TokenType.CodeBlock: + case token.TokenType.CodeBlock: type_str = "CodeBlock" - case TokenType.InlineCode: + case token.TokenType.InlineCode: type_str = "InlineCode" - case TokenType.HorizontalRule: + case token.TokenType.HorizontalRule: type_str = "HorizontalRule" - case TokenType.Underline: + case token.TokenType.Underline: type_str = "Underline" - case TokenType.Superscript: + case token.TokenType.Superscript: type_str = "Superscript" - case TokenType.Subscript: + case token.TokenType.Subscript: type_str = "Subscript" - case TokenType.HeaderLink: + case token.TokenType.HeaderLink: type_str = "HeaderLink" - case TokenType.Link: + case token.TokenType.Link: type_str = "Link" - case TokenType.Image: + case token.TokenType.Image: type_str = "Image" - case TokenType.Audio: + case token.TokenType.Audio: type_str = "Audio" - case TokenType.Video: + case token.TokenType.Video: type_str = "Video" - case TokenType.Iframe: + case token.TokenType.Iframe: type_str = "Iframe" - case TokenType.Script: + case token.TokenType.Script: type_str = "Script" - case TokenType.ScriptBlock: + case token.TokenType.ScriptBlock: type_str = "ScriptBlock" - case TokenType.Variable: + case token.TokenType.Variable: type_str = "Variable" - case TokenType.VariableDef: + case token.TokenType.VariableDef: type_str = "VariableDef" } value_str := "" - switch v in token.value { + switch v in tok.value { case string: value_str = v case int: @@ -108,13 +111,13 @@ print_token :: proc(token: ^Token, depth: int = 0) { } fmt.println() - for child in token.children { + for child in tok.children { print_token(child, depth + 1) } } main :: proc() { lines := read_stdin() - parsed := parse(lines) - print_html(&parsed) + parsed := parser.parse(lines) + renderer.print_html(&parsed) } diff --git a/src/embed.odin b/src/mime/mime.odin similarity index 99% rename from src/embed.odin rename to src/mime/mime.odin index 9136036..8e36d36 100644 --- a/src/embed.odin +++ b/src/mime/mime.odin @@ -1,4 +1,4 @@ -package main +package mime import "core:strings" diff --git a/src/parser.odin b/src/parser.odin deleted file mode 100644 index efbd97c..0000000 --- a/src/parser.odin +++ /dev/null @@ -1,1146 +0,0 @@ -package main - -import "core:strconv" -import "core:strings" - -TokenType :: enum { - Text, - Italics, - Bold, - BreakLine, - Header, - Paragraph, - Root, - Blockquote, - UnorderedList, - OrderedList, - ListItem, - Strikethrough, - Table, - TableRow, - TableHeaderRow, - TableCell, - TableSeparator, - Left, - Right, - Middle, - FootnoteDef, - FootnoteRef, - CodeBlock, - InlineCode, - HorizontalRule, - Underline, - Superscript, - Subscript, - HeaderLink, - Link, - Image, - Audio, - Video, - Iframe, - Script, - ScriptBlock, - Variable, - VariableDef, -} - -TokenValue :: union { - string, - int, -} - -Token :: struct { - type: TokenType, - value: TokenValue, - children: [dynamic]^Token, - parent: ^Token, -} - -push_token :: proc(parent: ^Token, type: TokenType, value: TokenValue = nil) -> ^Token { - token := new(Token) - token^ = Token{type, value, nil, parent} - append(&parent.children, token) - return token -} - -is_table_separator :: proc(line: string) -> bool { - inner := line[1:len(line) - 1] - cells := strings.split(inner, "|", context.temp_allocator) - if len(cells) == 0 { - return false - } - for cell in cells { - trimmed := strings.trim_space(cell) - if len(trimmed) == 0 { - return false - } - for ch in trimmed { - if ch != '-' { - return false - } - } - } - return true -} - -parse_table_cells :: proc(line: string) -> []string { - inner := line[1:len(line) - 1] - cells := strings.split(inner, "|", context.temp_allocator) - result := make([]string, len(cells), context.temp_allocator) - for cell, i in cells { - result[i] = strings.trim_space(cell) - } - return result -} - -get_alignment_token :: proc(text: string) -> (TokenType, bool) { - trimmed := strings.trim_space(text) - if len(trimmed) == 0 { - return .Text, false - } - if strings.has_suffix(trimmed, ">>") || strings.has_prefix(trimmed, ">>") { - return .Right, true - } - if strings.has_suffix(trimmed, "<<") || strings.has_prefix(trimmed, "<<") { - return .Left, true - } - if strings.has_suffix(trimmed, "^^") || strings.has_prefix(trimmed, "^^") { - return .Middle, true - } - return .Text, false -} - -push_text :: proc(parent: ^Token, text: string) { - align_type, has_align := get_alignment_token(text) - cleaned := strip_alignment_markers(text) - - target := parent - if has_align { - target = push_token(parent, align_type, nil) - } - push_token(target, TokenType.Text, cleaned) -} - -push_text_line :: proc(target: ^Token, text: string) { - if len(target.children) > 0 { - br := new(Token) - br^ = Token{TokenType.BreakLine, nil, nil, target} - append(&target.children, br) - } - push_text(target, text) -} - -strip_alignment_markers :: proc(text: string) -> string { - trimmed := strings.trim_space(text) - if strings.has_suffix(trimmed, ">>") { - return strings.trim_space(trimmed[:len(trimmed) - 2]) - } - if strings.has_prefix(trimmed, ">>") { - return strings.trim_space(trimmed[2:]) - } - if strings.has_suffix(trimmed, "<<") { - return strings.trim_space(trimmed[:len(trimmed) - 2]) - } - if strings.has_prefix(trimmed, "<<") { - return strings.trim_space(trimmed[2:]) - } - if strings.has_suffix(trimmed, "^^") { - return strings.trim_space(trimmed[:len(trimmed) - 2]) - } - if strings.has_prefix(trimmed, "^^") { - return strings.trim_space(trimmed[2:]) - } - return trimmed -} - -ESCAPE_PLACEHOLDER_BASE :: 0xE000 // Unicode Private Use Area - -is_escapable_char :: proc(b: byte) -> bool { - switch b { - case '\\', - '*', - '_', - '~', - '^', - '`', - '[', - ']', - '(', - ')', - '$', - '&', - '!', - '>', - '|', - '-', - '=', - '#', - '.': - return true - } - return false -} - -escape_placeholder_rune :: proc(b: byte) -> rune { - return rune(ESCAPE_PLACEHOLDER_BASE + int(b)) -} - -is_escape_placeholder :: proc(r: rune) -> bool { - return r >= ESCAPE_PLACEHOLDER_BASE && r < ESCAPE_PLACEHOLDER_BASE + 256 -} - -escape_placeholder_to_byte :: proc(r: rune) -> byte { - return byte(int(r) - ESCAPE_PLACEHOLDER_BASE) -} - -unescape_backslashes :: proc(s: string, allocator := context.temp_allocator) -> string { - if !strings.contains(s, "\\") { - return s - } - - b := strings.builder_make(allocator) - i := 0 - for i < len(s) { - if s[i] == '\\' && i + 1 < len(s) && is_escapable_char(s[i + 1]) { - strings.write_rune(&b, escape_placeholder_rune(s[i + 1])) - i += 2 - } else { - strings.write_byte(&b, s[i]) - i += 1 - } - } - return strings.to_string(b) -} - -decode_escapes :: proc(s: string, allocator := context.temp_allocator) -> string { - has_placeholder := false - for r in s { - if is_escape_placeholder(r) { - has_placeholder = true - break - } - } - if !has_placeholder { - return s - } - - b := strings.builder_make(allocator) - for r in s { - if is_escape_placeholder(r) { - strings.write_byte(&b, escape_placeholder_to_byte(r)) - } else { - strings.write_rune(&b, r) - } - } - return strings.to_string(b) -} - -is_space_byte :: proc(b: byte) -> bool { - return b == ' ' || b == '\t' -} - -parse_header :: proc(line: string) -> (int, string) { - if len(line) == 0 { - return 0, line - } - - lvl := 0 - for lvl < len(line) && line[lvl] == '!' { - lvl += 1 - } - if lvl == 0 || lvl >= len(line) || line[lvl] != ' ' { - return 0, line - } - return lvl, line[lvl + 1:] -} - -parse_footnote_def :: proc(line: string) -> (string, string, bool) { - if strings.has_prefix(line, "[&&") { - idx := strings.index_byte(line, ']') - if idx != -1 { - id := line[3:idx] - text := strings.trim_space(line[idx + 1:]) - return id, text, true - } - } - return "", "", false -} - -StyleToggle :: struct { - amount: int, - type: TokenType, -} - -parse_inline_style :: proc(token: ^Token, symbol: byte, toggles: []StyleToggle) { - if token.children != nil { - for i := 0; i < len(token.children); i += 1 { - parse_inline_style(token.children[i], symbol, toggles) - } - } - - if token.type == .Text { - str, ok := token.value.(string) - if !ok { - return - } - - symbol_found := false - for i := 0; i < len(str); i += 1 { - if str[i] == symbol { - symbol_found = true - break - } - } - if !symbol_found { - return - } - - token.value = nil - - active_styles: [256]bool - current_node := token - - i := 0 - start := 0 - - for i < len(str) { - if str[i] == symbol { - if i > start { - push_token(current_node, TokenType.Text, str[start:i]) - } - - count := 0 - symbol_start := i - for i < len(str) && str[i] == symbol { - count += 1 - i += 1 - } - - preceded_by_space := symbol_start > 0 && is_space_byte(str[symbol_start - 1]) - followed_by_space := i < len(str) && is_space_byte(str[i]) - if preceded_by_space && followed_by_space { - push_token(current_node, TokenType.Text, str[symbol_start:i]) - start = i - continue - } - - for count > 0 { - matched := false - for toggle in toggles { - if count >= toggle.amount { - active_styles[int(toggle.type)] = !active_styles[int(toggle.type)] - count -= toggle.amount - matched = true - break - } - } - if !matched { - push_token( - current_node, - TokenType.Text, - str[symbol_start:symbol_start + count], - ) - count = 0 - break - } - } - - current_node = token - for toggle in toggles { - if active_styles[int(toggle.type)] { - last := - current_node.children[len(current_node.children) - 1] if len(current_node.children) > 0 else nil - current_node = - last if last != nil && last.type == toggle.type else push_token(current_node, toggle.type, nil) - } - } - - start = i - } else { - i += 1 - } - } - - if i > start { - push_token(current_node, TokenType.Text, str[start:i]) - } - } -} - -pass_through_children :: proc(token: ^Token) { - if token.type == .CodeBlock || token.type == .InlineCode { - return - } - parse_inline_style(token, '*', []StyleToggle{{2, .Bold}, {1, .Italics}}) - parse_inline_style(token, '~', []StyleToggle{{2, .Strikethrough}}) - parse_inline_style(token, '-', []StyleToggle{{2, .Strikethrough}}) - parse_inline_style(token, '^', []StyleToggle{{1, .Superscript}}) - parse_inline_style(token, '_', []StyleToggle{{2, .Underline}, {1, .Subscript}}) -} - -parse_inline_code :: proc(token: ^Token) { - if token.type == .CodeBlock || token.type == .InlineCode { - return - } - if token.children != nil { - for i := 0; i < len(token.children); i += 1 { - parse_inline_code(token.children[i]) - } - } - - if token.type == .Text { - str, ok := token.value.(string) - if !ok { - return - } - - start_idx := strings.index_byte(str, '`') - if start_idx == -1 { - return - } - - end_idx := strings.index_byte(str[start_idx + 1:], '`') - if end_idx == -1 { - return - } - end_idx += start_idx + 1 - - code := str[start_idx + 1:end_idx] - - token.value = nil - - if start_idx > 0 { - push_token(token, .Text, str[:start_idx]) - } - push_token(token, .InlineCode, code) - - if end_idx + 1 < len(str) { - rem := push_token(token, .Text, str[end_idx + 1:]) - parse_inline_code(rem) - } - } -} - -parse_inline_footnotes :: proc(token: ^Token) { - if token.type == .CodeBlock || token.type == .InlineCode { - return - } - if token.children != nil { - for i := 0; i < len(token.children); i += 1 { - parse_inline_footnotes(token.children[i]) - } - } - - if token.type == .Text { - str, ok := token.value.(string) - if !ok { - return - } - - start_idx := strings.index(str, "[&") - if start_idx == -1 { - return - } - - end_idx := strings.index_byte(str[start_idx:], ']') - if end_idx == -1 { - return - } - end_idx += start_idx - - id := str[start_idx + 2:end_idx] - - token.value = nil - - if start_idx > 0 { - push_token(token, .Text, str[:start_idx]) - } - push_token(token, .FootnoteRef, id) - - if end_idx + 1 < len(str) { - rem := push_token(token, .Text, str[end_idx + 1:]) - parse_inline_footnotes(rem) - } - } -} - -match_pair :: proc(s: string, open1, close1, open2, close2: byte) -> (int, string, string, bool) { - if len(s) == 0 || s[0] != open1 { - return 0, "", "", false - } - - depth := 1 - close1_idx := -1 - for i := 1; i < len(s); i += 1 { - if s[i] == open1 { - depth += 1 - } else if s[i] == close1 { - depth -= 1 - if depth == 0 { - close1_idx = i - break - } - } - } - if close1_idx == -1 { - return 0, "", "", false - } - - if close1_idx + 1 < len(s) && s[close1_idx + 1] == open2 { - depth = 1 - close2_idx := -1 - for i := close1_idx + 2; i < len(s); i += 1 { - if s[i] == open2 { - depth += 1 - } else if s[i] == close2 { - depth -= 1 - if depth == 0 { - close2_idx = i - break - } - } - } - if close2_idx != -1 { - return close2_idx + 1, s[1:close1_idx], s[close1_idx + 2:close2_idx], true - } - } - return 0, "", "", false -} - -match_single :: proc(s: string, open, close: byte) -> (int, string, bool) { - if len(s) == 0 || s[0] != open { - return 0, "", false - } - close_idx := strings.index_byte(s, close) - if close_idx == -1 { - return 0, "", false - } - return close_idx + 1, s[1:close_idx], true -} - -find_link :: proc( - str: string, -) -> ( - start_idx, end_idx: int, - is_link: bool, - url: string, - alt: string, - is_header: bool, -) { - for i := 0; i < len(str); i += 1 { - if str[i] == '#' && i + 1 < len(str) { - if str[i + 1] == '[' { - if end, alt_text, url_text, ok := match_pair(str[i + 1:], '[', ']', '(', ')'); ok { - return i, i + 1 + end, true, url_text, alt_text, false - } - if end, inner, ok := match_single(str[i + 1:], '[', ']'); ok { - return i, i + 1 + end, true, inner, inner, false - } - } else if str[i + 1] == '(' { - if end, url_text, alt_text, ok := match_pair(str[i + 1:], '(', ')', '[', ']'); ok { - return i, i + 1 + end, true, url_text, alt_text, false - } - if end, inner, ok := match_single(str[i + 1:], '(', ')'); ok { - return i, i + 1 + end, true, inner, inner, false - } - } - } - - if str[i] == '[' { - if end, alt_text, url_text, ok := match_pair(str[i:], '[', ']', '(', ')'); ok { - if strings.has_prefix(url_text, "#") { - return i, i + end, true, url_text[1:], alt_text, true - } - } - if end, inner, ok := match_single(str[i:], '[', ']'); ok { - if strings.has_prefix(inner, "#") { - return i, i + end, true, inner[1:], inner[1:], true - } - } - } else if str[i] == '(' { - if end, url_text, alt_text, ok := match_pair(str[i:], '(', ')', '[', ']'); ok { - if strings.has_prefix(url_text, "#") { - return i, i + end, true, url_text[1:], alt_text, true - } - } - if end, inner, ok := match_single(str[i:], '(', ')'); ok { - if strings.has_prefix(inner, "#") { - return i, i + end, true, inner[1:], inner[1:], true - } - } - } - } - - return -1, -1, false, "", "", false -} - -parse_links :: proc(token: ^Token) { - if token.type == .CodeBlock || token.type == .InlineCode { - return - } - if token.children != nil { - for i := 0; i < len(token.children); i += 1 { - parse_links(token.children[i]) - } - } - - if token.type == .Text { - str, ok := token.value.(string) - if !ok { - return - } - - start_idx, end_idx, is_link, url, alt, is_header := find_link(str) - if !is_link { - return - } - - token.value = nil - - if start_idx > 0 { - push_token(token, .Text, str[:start_idx]) - } - - link_token := push_token(token, .HeaderLink if is_header else .Link, url) - push_token(link_token, .Text, alt) - - if end_idx < len(str) { - rem := push_token(token, .Text, str[end_idx:]) - parse_links(rem) - } - } -} - -find_script_block :: proc( - str: string, -) -> ( - start_idx, end_idx: int, - is_block: bool, - block_type: string, - content: string, -) { - for i := 0; i < len(str); i += 1 { - if str[i] == '$' && i + 1 < len(str) && str[i + 1] == '[' { - if end, inner, ok := match_single(str[i + 1:], '[', ']'); ok { - inner = strings.trim_space(inner) - if strings.has_prefix(inner, "/") { - continue - } - if inner == "script" || inner == "js" || inner == "javascript" { - b := strings.builder_make(context.temp_allocator) - strings.write_string(&b, "$[/") - strings.write_string(&b, inner) - strings.write_string(&b, "]") - close_tag := strings.to_string(b) - close_idx := strings.index(str[i + 1 + end:], close_tag) - if close_idx != -1 { - close_idx += i + 1 + end - content := str[i + 1 + end:close_idx] - return i, close_idx + len( - close_tag, - ), true, inner, strings.trim_space(content) - } - } - } - } - } - return -1, -1, false, "", "" -} - -find_embed :: proc( - str: string, -) -> ( - start_idx, end_idx: int, - is_embed: bool, - url: string, - alt: string, - embed_type: string, -) { - for i := 0; i < len(str); i += 1 { - if str[i] == '$' && i + 1 < len(str) { - if str[i + 1] == '[' { - if end, alt_text, url_text, ok := match_pair(str[i + 1:], '[', ']', '(', ')'); ok { - clean_url := url_text - type_hint := "" - if pipe_idx := strings.index_byte(url_text, '|'); pipe_idx != -1 { - clean_url = url_text[:pipe_idx] - type_hint = url_text[pipe_idx + 1:] - } - - if is_url(clean_url) { - e_type := "" - if type_hint != "" { - e_type = parse_type_hint(type_hint) - } - if e_type == "" { - e_type = detect_embed_type(clean_url) - } - return i, i + 1 + end, true, clean_url, alt_text, e_type - } - } - if end, inner, ok := match_single(str[i + 1:], '[', ']'); ok { - url_text := inner - type_hint := "" - if pipe_idx := strings.index_byte(inner, '|'); pipe_idx != -1 { - url_text = inner[:pipe_idx] - type_hint = inner[pipe_idx + 1:] - } - - if is_url(url_text) { - e_type := "" - if type_hint != "" { - e_type = parse_type_hint(type_hint) - } - if e_type == "" { - e_type = detect_embed_type(url_text) - } - return i, i + 1 + end, true, url_text, url_text, e_type - } - } - } else if str[i + 1] == '(' { - if end, url_text, alt_text, ok := match_pair(str[i + 1:], '(', ')', '[', ']'); ok { - clean_url := url_text - type_hint := "" - if pipe_idx := strings.index_byte(url_text, '|'); pipe_idx != -1 { - clean_url = url_text[:pipe_idx] - type_hint = url_text[pipe_idx + 1:] - } - - if is_url(clean_url) { - e_type := "" - if type_hint != "" { - e_type = parse_type_hint(type_hint) - } - if e_type == "" { - e_type = detect_embed_type(clean_url) - } - return i, i + 1 + end, true, clean_url, alt_text, e_type - } - } - if end, inner, ok := match_single(str[i + 1:], '(', ')'); ok { - url_text := inner - type_hint := "" - if pipe_idx := strings.index_byte(inner, '|'); pipe_idx != -1 { - url_text = inner[:pipe_idx] - type_hint = inner[pipe_idx + 1:] - } - - if is_url(url_text) { - e_type := "" - if type_hint != "" { - e_type = parse_type_hint(type_hint) - } - if e_type == "" { - e_type = detect_embed_type(url_text) - } - return i, i + 1 + end, true, url_text, url_text, e_type - } - } - } - } - } - - return -1, -1, false, "", "", "" -} - -parse_embeds :: proc(token: ^Token) { - if token.type == .CodeBlock || token.type == .InlineCode { - return - } - if token.children != nil { - for i := 0; i < len(token.children); i += 1 { - parse_embeds(token.children[i]) - } - } - - if token.type == .Text { - str, ok := token.value.(string) - if !ok { - return - } - - s_start, s_end, is_script, s_type, s_content := find_script_block(str) - if is_script { - token.value = nil - - if s_start > 0 { - push_token(token, .Text, str[:s_start]) - } - - script_token := push_token(token, .ScriptBlock, s_content) - - if s_end < len(str) { - rem := push_token(token, .Text, str[s_end:]) - parse_embeds(rem) - } - return - } - - start_idx, end_idx, is_embed, url, alt, e_type := find_embed(str) - if !is_embed { - return - } - - token.value = nil - - if start_idx > 0 { - push_token(token, .Text, str[:start_idx]) - } - - embed_token_type: TokenType - if e_type == "image" { - embed_token_type = .Image - } else if e_type == "audio" { - embed_token_type = .Audio - } else if e_type == "video" { - embed_token_type = .Video - } else if e_type == "script" { - embed_token_type = .Script - } else { - embed_token_type = .Iframe - } - - embed_token := push_token(token, embed_token_type, url) - push_token(embed_token, .Text, alt) - - if end_idx < len(str) { - rem := push_token(token, .Text, str[end_idx:]) - parse_embeds(rem) - } - } -} - -find_variable :: proc( - str: string, -) -> ( - start_idx, end_idx: int, - is_var: bool, - name: string, - val: string, - is_def: bool, -) { - for i := 0; i < len(str); i += 1 { - if str[i] == '$' && i + 1 < len(str) && str[i + 1] == '(' { - if end, inner, ok := match_single(str[i + 1:], '(', ')'); ok { - eq := strings.index_byte(inner, '=') - if eq != -1 { - return i, - i + 1 + end, - true, - strings.trim_space(inner[:eq]), - strings.trim_space(inner[eq + 1:]), - true - } else if is_url(inner) { - continue - } else { - return i, i + 1 + end, true, strings.trim_space(inner), "", false - } - } - } - } - - return -1, -1, false, "", "", false -} - -parse_variables :: proc(token: ^Token) { - if token.type == .CodeBlock || token.type == .InlineCode { - return - } - if token.children != nil { - for i := 0; i < len(token.children); i += 1 { - parse_variables(token.children[i]) - } - } - - if token.type == .Text { - str, ok := token.value.(string) - if !ok { - return - } - - start_idx, end_idx, is_var, name, val, is_def := find_variable(str) - if !is_var { - return - } - - token.value = nil - - if start_idx > 0 { - push_token(token, .Text, str[:start_idx]) - } - - if is_def { - def_token := push_token(token, .VariableDef, name) - push_token(def_token, .Text, val) - } else { - push_token(token, .Variable, name) - } - - if end_idx < len(str) { - rem := push_token(token, .Text, str[end_idx:]) - parse_variables(rem) - } - } -} - -resolve_variables :: proc(token: ^Token, vars: ^map[string]string) { - if token.type == .VariableDef { - if name, ok := token.value.(string); ok { - if len(token.children) > 0 { - if val, vok := token.children[0].value.(string); vok { - vars[name] = val - } - } - } - return - } - if token.type == .Variable { - if name, ok := token.value.(string); ok { - if val, found := vars[name]; found { - if token.children == nil { - push_token(token, .Text, val) - } - } - } - return - } - if token.type == .Link || token.type == .HeaderLink { - if url, ok := token.value.(string); ok { - for i := 0; i < len(url); i += 1 { - if url[i] == '$' && i + 1 < len(url) && url[i + 1] == '(' { - if end, inner, ok := match_single(url[i + 1:], '(', ')'); ok { - var_name := strings.trim_space(inner) - if val, found := vars[var_name]; found { - b := strings.builder_make(context.temp_allocator) - strings.write_string(&b, url[:i]) - strings.write_string(&b, val) - strings.write_string(&b, url[i + 1 + end:]) - token.value = strings.to_string(b) - break - } - } - } - } - } - } - if token.children != nil { - for child in token.children { - resolve_variables(child, vars) - } - } -} - -parse :: proc(lines: []string, allocator := context.allocator) -> Token { - root := Token{TokenType.Root, nil, nil, nil} - current_token := &root - - // FIRST PASS - - in_code_block := false - in_script_block := false - script_block_content: [dynamic]string - script_block_type := "" - - for untrimmed_line, line_index in lines { - line := strings.trim_space(untrimmed_line) - - if in_code_block { - if strings.has_prefix(line, "```") { - in_code_block = false - current_token = &root - } else { - push_token(current_token, TokenType.Text, untrimmed_line) - } - continue - } - - if in_script_block { - if strings.has_prefix(line, "$[/") { - full_content := strings.join(script_block_content[:], "\n") - script_token := push_token(&root, TokenType.ScriptBlock, full_content) - push_token(script_token, TokenType.Text, script_block_type) - in_script_block = false - script_block_content = nil - script_block_type = "" - current_token = &root - } else { - append(&script_block_content, untrimmed_line) - } - continue - } - - line = unescape_backslashes(line) - - if strings.has_prefix(line, "```") { - lang := strings.trim_space(line[3:]) - current_token = push_token(&root, TokenType.CodeBlock, lang) - in_code_block = true - continue - } - - if strings.has_prefix(line, "$[") && strings.has_suffix(line, "]") { - inner := strings.trim_space(line[2:len(line) - 1]) - if inner == "script" || inner == "js" || inner == "javascript" { - in_script_block = true - script_block_type = inner - script_block_content = nil - current_token = &root - continue - } - } - - if len(line) >= 3 { - is_hr := true - ch := line[0] - if ch == '-' || ch == '_' || ch == '=' { - for i := 1; i < len(line); i += 1 { - if line[i] != ch { - is_hr = false - break - } - } - } else { - is_hr = false - } - - if is_hr { - push_token(&root, TokenType.HorizontalRule, nil) - current_token = &root - continue - } - } - - if len(line) == 0 { - current_token = &root - continue - } - - header_level, text := parse_header(line) - if header_level > 0 { - current_token = push_token(&root, TokenType.Header, header_level) - push_text(current_token, text) - current_token = &root - continue - } - - if id, ftext, ok := parse_footnote_def(line); ok { - current_token = push_token(&root, TokenType.FootnoteDef, id) - push_text(current_token, ftext) - current_token = &root - continue - } - - if strings.has_prefix(text, "> ") { - if current_token.type != TokenType.Blockquote { - current_token = push_token(&root, TokenType.Blockquote, nil) - push_token(current_token, TokenType.Paragraph, nil) - } - last_p := current_token.children[len(current_token.children) - 1] - push_text_line(last_p, text[2:]) - continue - } - - if strings.has_prefix(text, ". ") || strings.has_prefix(text, "- ") { - if current_token.type != TokenType.UnorderedList { - current_token = push_token(&root, TokenType.UnorderedList, nil) - } - li := push_token(current_token, TokenType.ListItem, nil) - push_text(li, text[2:]) - continue - } - - if len(text) >= 3 { - item := 1 - idx := 0 - for idx < len(text) && text[idx] >= '0' && text[idx] <= '9' { - idx += 1 - } - - if idx > 0 && idx + 1 < len(text) && text[idx] == '.' && text[idx + 1] == ' ' { - item, _ = strconv.parse_int(text[:idx]) - text = text[idx + 2:] - - if current_token.type != TokenType.OrderedList { - current_token = push_token(&root, TokenType.OrderedList, item) - } - li := push_token(current_token, TokenType.ListItem, nil) - push_text(li, text) - continue - } - } - - if len(text) >= 3 && text[0] == '|' && text[len(text) - 1] == '|' { - if current_token.type != TokenType.Table { - current_token = push_token(&root, TokenType.Table, nil) - } - - if is_table_separator(text) { - row_count := len(current_token.children) - if row_count == 1 { - current_token.children[0].type = TokenType.TableHeaderRow - } else if row_count > 1 { - push_token(current_token, TokenType.TableSeparator, nil) - } - } else { - row := push_token(current_token, TokenType.TableRow, nil) - cells := parse_table_cells(text) - for cell_text in cells { - cell := push_token(row, TokenType.TableCell, nil) - push_text(cell, cell_text) - } - } - continue - } - - // LAST-CASE SCENARIO: NORMAL TEXT - - if current_token.type == .Blockquote { - last_p := current_token.children[len(current_token.children) - 1] - push_text_line(last_p, line) - } else if current_token.type == .UnorderedList || current_token.type == .OrderedList { - last_li := current_token.children[len(current_token.children) - 1] - push_text_line(last_li, line) - } else { - if current_token.type != .Paragraph { - current_token = push_token(&root, TokenType.Paragraph, nil) - } - push_text_line(current_token, line) - } - } - - // SECOND PASS - - for token in root.children { - parse_inline_code(token) - parse_inline_footnotes(token) - parse_links(token) - parse_variables(token) - parse_embeds(token) - pass_through_children(token) - } - - // VARIABLE RESOLUTION PASS - - vars: map[string]string - for child in root.children { - resolve_variables(child, &vars) - } - - // FOURTH PASS - - footnotes: [dynamic]^Token - new_root_children: [dynamic]^Token - for child in root.children { - if child.type == .FootnoteDef { - append(&footnotes, child) - } else { - append(&new_root_children, child) - } - } - for child in footnotes { - append(&new_root_children, child) - } - root.children = new_root_children - - return root -} diff --git a/src/parser/embed.odin b/src/parser/embed.odin new file mode 100644 index 0000000..5295bb5 --- /dev/null +++ b/src/parser/embed.odin @@ -0,0 +1,204 @@ +package parser + +import "../mime" +import "../token" +import "core:strings" + +find_script_block :: proc( + str: string, +) -> ( + start_idx, end_idx: int, + is_block: bool, + block_type: string, + content: string, +) { + for i := 0; i < len(str); i += 1 { + if str[i] == '$' && i + 1 < len(str) && str[i + 1] == '[' { + if end, inner, ok := match_single(str[i + 1:], '[', ']'); ok { + inner = strings.trim_space(inner) + if strings.has_prefix(inner, "/") { + continue + } + if inner == "script" || inner == "js" || inner == "javascript" { + b := strings.builder_make(context.temp_allocator) + strings.write_string(&b, "$[/") + strings.write_string(&b, inner) + strings.write_string(&b, "]") + close_tag := strings.to_string(b) + close_idx := strings.index(str[i + 1 + end:], close_tag) + if close_idx != -1 { + close_idx += i + 1 + end + content := str[i + 1 + end:close_idx] + return i, close_idx + len( + close_tag, + ), true, inner, strings.trim_space(content) + } + } + } + } + } + return -1, -1, false, "", "" +} + +find_embed :: proc( + str: string, +) -> ( + start_idx, end_idx: int, + is_embed: bool, + url: string, + alt: string, + embed_type: string, +) { + for i := 0; i < len(str); i += 1 { + if str[i] == '$' && i + 1 < len(str) { + if str[i + 1] == '[' { + if end, alt_text, url_text, ok := match_pair(str[i + 1:], '[', ']', '(', ')'); ok { + clean_url := url_text + type_hint := "" + if pipe_idx := strings.index_byte(url_text, '|'); pipe_idx != -1 { + clean_url = url_text[:pipe_idx] + type_hint = url_text[pipe_idx + 1:] + } + + if mime.is_url(clean_url) { + e_type := "" + if type_hint != "" { + e_type = mime.parse_type_hint(type_hint) + } + if e_type == "" { + e_type = mime.detect_embed_type(clean_url) + } + return i, i + 1 + end, true, clean_url, alt_text, e_type + } + } + if end, inner, ok := match_single(str[i + 1:], '[', ']'); ok { + url_text := inner + type_hint := "" + if pipe_idx := strings.index_byte(inner, '|'); pipe_idx != -1 { + url_text = inner[:pipe_idx] + type_hint = inner[pipe_idx + 1:] + } + + if mime.is_url(url_text) { + e_type := "" + if type_hint != "" { + e_type = mime.parse_type_hint(type_hint) + } + if e_type == "" { + e_type = mime.detect_embed_type(url_text) + } + return i, i + 1 + end, true, url_text, url_text, e_type + } + } + } else if str[i + 1] == '(' { + if end, url_text, alt_text, ok := match_pair(str[i + 1:], '(', ')', '[', ']'); ok { + clean_url := url_text + type_hint := "" + if pipe_idx := strings.index_byte(url_text, '|'); pipe_idx != -1 { + clean_url = url_text[:pipe_idx] + type_hint = url_text[pipe_idx + 1:] + } + + if mime.is_url(clean_url) { + e_type := "" + if type_hint != "" { + e_type = mime.parse_type_hint(type_hint) + } + if e_type == "" { + e_type = mime.detect_embed_type(clean_url) + } + return i, i + 1 + end, true, clean_url, alt_text, e_type + } + } + if end, inner, ok := match_single(str[i + 1:], '(', ')'); ok { + url_text := inner + type_hint := "" + if pipe_idx := strings.index_byte(inner, '|'); pipe_idx != -1 { + url_text = inner[:pipe_idx] + type_hint = inner[pipe_idx + 1:] + } + + if mime.is_url(url_text) { + e_type := "" + if type_hint != "" { + e_type = mime.parse_type_hint(type_hint) + } + if e_type == "" { + e_type = mime.detect_embed_type(url_text) + } + return i, i + 1 + end, true, url_text, url_text, e_type + } + } + } + } + } + + return -1, -1, false, "", "", "" +} + +parse_embeds :: proc(tok: ^token.Token) { + if tok.type == .CodeBlock || tok.type == .InlineCode { + return + } + if tok.children != nil { + for i := 0; i < len(tok.children); i += 1 { + parse_embeds(tok.children[i]) + } + } + + if tok.type == .Text { + str, ok := tok.value.(string) + if !ok { + return + } + + s_start, s_end, is_script, s_type, s_content := find_script_block(str) + if is_script { + tok.value = nil + + if s_start > 0 { + push_token(tok, .Text, str[:s_start]) + } + + script_token := push_token(tok, .ScriptBlock, s_content) + + if s_end < len(str) { + rem := push_token(tok, .Text, str[s_end:]) + parse_embeds(rem) + } + return + } + + start_idx, end_idx, is_embed, url, alt, e_type := find_embed(str) + if !is_embed { + return + } + + tok.value = nil + + if start_idx > 0 { + push_token(tok, .Text, str[:start_idx]) + } + + embed_token_type: token.TokenType + if e_type == "image" { + embed_token_type = .Image + } else if e_type == "audio" { + embed_token_type = .Audio + } else if e_type == "video" { + embed_token_type = .Video + } else if e_type == "script" { + embed_token_type = .Script + } else { + embed_token_type = .Iframe + } + + embed_token := push_token(tok, embed_token_type, url) + push_token(embed_token, .Text, alt) + + if end_idx < len(str) { + rem := push_token(tok, .Text, str[end_idx:]) + parse_embeds(rem) + } + } +} diff --git a/src/parser/escape.odin b/src/parser/escape.odin new file mode 100644 index 0000000..f0e768a --- /dev/null +++ b/src/parser/escape.odin @@ -0,0 +1,124 @@ +package parser + +import "../token" +import "core:strings" + +ESCAPE_PLACEHOLDER_BASE :: 0xE000 + +is_escapable_char :: proc(b: byte) -> bool { + switch b { + case '\\', + '*', + '_', + '~', + '^', + '`', + '[', + ']', + '(', + ')', + '$', + '&', + '!', + '>', + '<', + '|', + '-', + '=', + '#', + '.': + return true + } + return false +} + +is_repeatable_token_char :: proc(b: byte) -> bool { + switch b { + case '*', '_', '~', '^', '<', '>', '-', '=', '&', '!', '.': + return true + } + return false +} + +escape_placeholder_rune :: proc(b: byte) -> rune { + return rune(ESCAPE_PLACEHOLDER_BASE + int(b)) +} + +is_escape_placeholder :: proc(r: rune) -> bool { + return r >= ESCAPE_PLACEHOLDER_BASE && r < ESCAPE_PLACEHOLDER_BASE + 256 +} + +escape_placeholder_to_byte :: proc(r: rune) -> byte { + return byte(int(r) - ESCAPE_PLACEHOLDER_BASE) +} + +unescape_backslashes :: proc(s: string, allocator := context.temp_allocator) -> string { + if !strings.contains(s, "\\") { + return s + } + + b := strings.builder_make(allocator) + i := 0 + for i < len(s) { + if s[i] == '`' { + close := strings.index_byte(s[i + 1:], '`') + if close != -1 { + end := i + 1 + close + strings.write_string(&b, s[i:end + 1]) + i = end + 1 + continue + } + } + if s[i] == '\\' && i + 1 < len(s) && is_escapable_char(s[i + 1]) { + esc := s[i + 1] + strings.write_rune(&b, escape_placeholder_rune(esc)) + i += 2 + if is_repeatable_token_char(esc) { + for i < len(s) && s[i] == esc { + strings.write_rune(&b, escape_placeholder_rune(esc)) + i += 1 + } + } + } else { + strings.write_byte(&b, s[i]) + i += 1 + } + } + return strings.to_string(b) +} + +decode_escapes :: proc(s: string, allocator := context.temp_allocator) -> string { + has_placeholder := false + for r in s { + if is_escape_placeholder(r) { + has_placeholder = true + break + } + } + if !has_placeholder { + return s + } + + b := strings.builder_make(allocator) + for r in s { + if is_escape_placeholder(r) { + strings.write_byte(&b, escape_placeholder_to_byte(r)) + } else { + strings.write_rune(&b, r) + } + } + return strings.to_string(b) +} + +decode_text_escapes :: proc(tok: ^token.Token) { + if tok.type == .Text { + if str, ok := tok.value.(string); ok { + tok.value = decode_escapes(str) + } + } + if tok.children != nil { + for child in tok.children { + decode_text_escapes(child) + } + } +} diff --git a/src/parser/inline.odin b/src/parser/inline.odin new file mode 100644 index 0000000..9ef7157 --- /dev/null +++ b/src/parser/inline.odin @@ -0,0 +1,470 @@ +package parser + +import "../mime" +import "../token" +import "core:strings" + +StyleToggle :: struct { + amount: int, + type: token.TokenType, +} + +parse_inline_style :: proc(tok: ^token.Token, symbol: byte, toggles: []StyleToggle) { + if tok.children != nil { + for i := 0; i < len(tok.children); i += 1 { + parse_inline_style(tok.children[i], symbol, toggles) + } + } + + if tok.type == .Text { + str, ok := tok.value.(string) + if !ok { + return + } + + symbol_found := false + for i := 0; i < len(str); i += 1 { + if str[i] == symbol { + symbol_found = true + break + } + } + if !symbol_found { + return + } + + tok.value = nil + + active_styles: [256]bool + current_node := tok + + i := 0 + start := 0 + + for i < len(str) { + if str[i] == symbol { + if i > start { + push_token(current_node, token.TokenType.Text, str[start:i]) + } + + count := 0 + symbol_start := i + for i < len(str) && str[i] == symbol { + count += 1 + i += 1 + } + + preceded_by_space := symbol_start > 0 && is_space_byte(str[symbol_start - 1]) + followed_by_space := i < len(str) && is_space_byte(str[i]) + if preceded_by_space && followed_by_space { + push_token(current_node, token.TokenType.Text, str[symbol_start:i]) + start = i + continue + } + + for count > 0 { + matched := false + for toggle in toggles { + if count >= toggle.amount { + active_styles[int(toggle.type)] = !active_styles[int(toggle.type)] + count -= toggle.amount + matched = true + break + } + } + if !matched { + push_token( + current_node, + token.TokenType.Text, + str[symbol_start:symbol_start + count], + ) + count = 0 + break + } + } + + current_node = tok + for toggle in toggles { + if active_styles[int(toggle.type)] { + last := + current_node.children[len(current_node.children) - 1] if len(current_node.children) > 0 else nil + current_node = + last if last != nil && last.type == toggle.type else push_token(current_node, toggle.type, nil) + } + } + + start = i + } else { + i += 1 + } + } + + if i > start { + push_token(current_node, token.TokenType.Text, str[start:i]) + } + } +} + +parse_inline_code :: proc(tok: ^token.Token) { + if tok.type == .CodeBlock || tok.type == .InlineCode { + return + } + if tok.children != nil { + for i := 0; i < len(tok.children); i += 1 { + parse_inline_code(tok.children[i]) + } + } + + if tok.type == .Text { + str, ok := tok.value.(string) + if !ok { + return + } + + start_idx := strings.index_byte(str, '`') + if start_idx == -1 { + return + } + + end_idx := strings.index_byte(str[start_idx + 1:], '`') + if end_idx == -1 { + return + } + end_idx += start_idx + 1 + + code := str[start_idx + 1:end_idx] + + tok.value = nil + + if start_idx > 0 { + push_token(tok, .Text, str[:start_idx]) + } + push_token(tok, .InlineCode, code) + + if end_idx + 1 < len(str) { + rem := push_token(tok, .Text, str[end_idx + 1:]) + parse_inline_code(rem) + } + } +} + +parse_inline_footnotes :: proc(tok: ^token.Token) { + if tok.type == .CodeBlock || tok.type == .InlineCode { + return + } + if tok.children != nil { + for i := 0; i < len(tok.children); i += 1 { + parse_inline_footnotes(tok.children[i]) + } + } + + if tok.type == .Text { + str, ok := tok.value.(string) + if !ok { + return + } + + start_idx := strings.index(str, "[&") + if start_idx == -1 { + return + } + + end_idx := strings.index_byte(str[start_idx:], ']') + if end_idx == -1 { + return + } + end_idx += start_idx + + id := str[start_idx + 2:end_idx] + + tok.value = nil + + if start_idx > 0 { + push_token(tok, .Text, str[:start_idx]) + } + push_token(tok, .FootnoteRef, id) + + if end_idx + 1 < len(str) { + rem := push_token(tok, .Text, str[end_idx + 1:]) + parse_inline_footnotes(rem) + } + } +} + +match_pair :: proc(s: string, open1, close1, open2, close2: byte) -> (int, string, string, bool) { + if len(s) == 0 || s[0] != open1 { + return 0, "", "", false + } + + depth := 1 + close1_idx := -1 + for i := 1; i < len(s); i += 1 { + if s[i] == open1 { + depth += 1 + } else if s[i] == close1 { + depth -= 1 + if depth == 0 { + close1_idx = i + break + } + } + } + if close1_idx == -1 { + return 0, "", "", false + } + + if close1_idx + 1 < len(s) && s[close1_idx + 1] == open2 { + depth = 1 + close2_idx := -1 + for i := close1_idx + 2; i < len(s); i += 1 { + if s[i] == open2 { + depth += 1 + } else if s[i] == close2 { + depth -= 1 + if depth == 0 { + close2_idx = i + break + } + } + } + if close2_idx != -1 { + return close2_idx + 1, s[1:close1_idx], s[close1_idx + 2:close2_idx], true + } + } + return 0, "", "", false +} + +match_single :: proc(s: string, open, close: byte) -> (int, string, bool) { + if len(s) == 0 || s[0] != open { + return 0, "", false + } + close_idx := strings.index_byte(s, close) + if close_idx == -1 { + return 0, "", false + } + return close_idx + 1, s[1:close_idx], true +} + +find_link :: proc( + str: string, +) -> ( + start_idx, end_idx: int, + is_link: bool, + url: string, + alt: string, + is_header: bool, +) { + for i := 0; i < len(str); i += 1 { + if str[i] == '#' && i + 1 < len(str) { + if str[i + 1] == '[' { + if end, alt_text, url_text, ok := match_pair(str[i + 1:], '[', ']', '(', ')'); ok { + return i, i + 1 + end, true, url_text, alt_text, false + } + if end, inner, ok := match_single(str[i + 1:], '[', ']'); ok { + return i, i + 1 + end, true, inner, inner, false + } + } else if str[i + 1] == '(' { + if end, url_text, alt_text, ok := match_pair(str[i + 1:], '(', ')', '[', ']'); ok { + return i, i + 1 + end, true, url_text, alt_text, false + } + if end, inner, ok := match_single(str[i + 1:], '(', ')'); ok { + return i, i + 1 + end, true, inner, inner, false + } + } + } + + if str[i] == '[' { + if end, alt_text, url_text, ok := match_pair(str[i:], '[', ']', '(', ')'); ok { + if strings.has_prefix(url_text, "#") { + return i, i + end, true, url_text[1:], alt_text, true + } + } + if end, inner, ok := match_single(str[i:], '[', ']'); ok { + if strings.has_prefix(inner, "#") { + return i, i + end, true, inner[1:], inner[1:], true + } + } + } else if str[i] == '(' { + if end, url_text, alt_text, ok := match_pair(str[i:], '(', ')', '[', ']'); ok { + if strings.has_prefix(url_text, "#") { + return i, i + end, true, url_text[1:], alt_text, true + } + } + if end, inner, ok := match_single(str[i:], '(', ')'); ok { + if strings.has_prefix(inner, "#") { + return i, i + end, true, inner[1:], inner[1:], true + } + } + } + } + + return -1, -1, false, "", "", false +} + +parse_links :: proc(tok: ^token.Token) { + if tok.type == .CodeBlock || tok.type == .InlineCode { + return + } + if tok.children != nil { + for i := 0; i < len(tok.children); i += 1 { + parse_links(tok.children[i]) + } + } + + if tok.type == .Text { + str, ok := tok.value.(string) + if !ok { + return + } + + start_idx, end_idx, is_link, url, alt, is_header := find_link(str) + if !is_link { + return + } + + tok.value = nil + + if start_idx > 0 { + push_token(tok, .Text, str[:start_idx]) + } + + link_token := push_token(tok, .HeaderLink if is_header else .Link, url) + push_token(link_token, .Text, alt) + + if end_idx < len(str) { + rem := push_token(tok, .Text, str[end_idx:]) + parse_links(rem) + } + } +} + +find_variable :: proc( + str: string, +) -> ( + start_idx, end_idx: int, + is_var: bool, + name: string, + val: string, + is_def: bool, +) { + for i := 0; i < len(str); i += 1 { + if str[i] == '$' && i + 1 < len(str) && str[i + 1] == '(' { + if end, inner, ok := match_single(str[i + 1:], '(', ')'); ok { + eq := strings.index_byte(inner, '=') + if eq != -1 { + return i, + i + 1 + end, + true, + strings.trim_space(inner[:eq]), + strings.trim_space(inner[eq + 1:]), + true + } else if mime.is_url(inner) { + continue + } else { + return i, i + 1 + end, true, strings.trim_space(inner), "", false + } + } + } + } + + return -1, -1, false, "", "", false +} + +parse_variables :: proc(tok: ^token.Token) { + if tok.type == .CodeBlock || tok.type == .InlineCode { + return + } + if tok.children != nil { + for i := 0; i < len(tok.children); i += 1 { + parse_variables(tok.children[i]) + } + } + + if tok.type == .Text { + str, ok := tok.value.(string) + if !ok { + return + } + + start_idx, end_idx, is_var, name, val, is_def := find_variable(str) + if !is_var { + return + } + + tok.value = nil + + if start_idx > 0 { + push_token(tok, .Text, str[:start_idx]) + } + + if is_def { + def_token := push_token(tok, .VariableDef, name) + push_token(def_token, .Text, val) + } else { + push_token(tok, .Variable, name) + } + + if end_idx < len(str) { + rem := push_token(tok, .Text, str[end_idx:]) + parse_variables(rem) + } + } +} + +resolve_variables :: proc(tok: ^token.Token, vars: ^map[string]string) { + if tok.type == .VariableDef { + if name, ok := tok.value.(string); ok { + if len(tok.children) > 0 { + if val, vok := tok.children[0].value.(string); vok { + vars[name] = val + } + } + } + return + } + if tok.type == .Variable { + if name, ok := tok.value.(string); ok { + if val, found := vars[name]; found { + if tok.children == nil { + push_token(tok, .Text, val) + } + } + } + return + } + if tok.type == .Link || tok.type == .HeaderLink { + if url, ok := tok.value.(string); ok { + for i := 0; i < len(url); i += 1 { + if url[i] == '$' && i + 1 < len(url) && url[i + 1] == '(' { + if end, inner, ok := match_single(url[i + 1:], '(', ')'); ok { + var_name := strings.trim_space(inner) + if val, found := vars[var_name]; found { + b := strings.builder_make(context.temp_allocator) + strings.write_string(&b, url[:i]) + strings.write_string(&b, val) + strings.write_string(&b, url[i + 1 + end:]) + tok.value = strings.to_string(b) + break + } + } + } + } + } + } + if tok.children != nil { + for child in tok.children { + resolve_variables(child, vars) + } + } +} + +pass_through_children :: proc(tok: ^token.Token) { + if tok.type == .CodeBlock || tok.type == .InlineCode { + return + } + parse_inline_style(tok, '*', []StyleToggle{{2, .Bold}, {1, .Italics}}) + parse_inline_style(tok, '~', []StyleToggle{{2, .Strikethrough}}) + parse_inline_style(tok, '-', []StyleToggle{{2, .Strikethrough}}) + parse_inline_style(tok, '^', []StyleToggle{{1, .Superscript}}) + parse_inline_style(tok, '_', []StyleToggle{{2, .Underline}, {1, .Subscript}}) +} diff --git a/src/parser/parser.odin b/src/parser/parser.odin new file mode 100644 index 0000000..c993b2b --- /dev/null +++ b/src/parser/parser.odin @@ -0,0 +1,211 @@ +package parser + +import "../token" +import "core:strconv" +import "core:strings" + +parse :: proc(lines: []string, allocator := context.allocator) -> token.Token { + root := token.Token{token.TokenType.Root, nil, nil, nil} + current_token := &root + + in_code_block := false + in_script_block := false + script_block_content: [dynamic]string + script_block_type := "" + + for untrimmed_line, line_index in lines { + line := strings.trim_space(untrimmed_line) + + if in_code_block { + if strings.has_prefix(line, "```") { + in_code_block = false + current_token = &root + } else { + push_token(current_token, token.TokenType.Text, untrimmed_line) + } + continue + } + + if in_script_block { + if strings.has_prefix(line, "$[/") { + full_content := strings.join(script_block_content[:], "\n") + script_token := push_token(&root, token.TokenType.ScriptBlock, full_content) + push_token(script_token, token.TokenType.Text, script_block_type) + in_script_block = false + script_block_content = nil + script_block_type = "" + current_token = &root + } else { + append(&script_block_content, untrimmed_line) + } + continue + } + + line = unescape_backslashes(line) + + if strings.has_prefix(line, "```") { + lang := strings.trim_space(line[3:]) + current_token = push_token(&root, token.TokenType.CodeBlock, lang) + in_code_block = true + continue + } + + if strings.has_prefix(line, "$[") && strings.has_suffix(line, "]") { + inner := strings.trim_space(line[2:len(line) - 1]) + if inner == "script" || inner == "js" || inner == "javascript" { + in_script_block = true + script_block_type = inner + script_block_content = nil + current_token = &root + continue + } + } + + if len(line) >= 3 { + is_hr := true + ch := line[0] + if ch == '-' || ch == '_' || ch == '=' { + for i := 1; i < len(line); i += 1 { + if line[i] != ch { + is_hr = false + break + } + } + } else { + is_hr = false + } + + if is_hr { + push_token(&root, token.TokenType.HorizontalRule, nil) + current_token = &root + continue + } + } + + if len(line) == 0 { + current_token = &root + continue + } + + header_level, text := parse_header(line) + if header_level > 0 { + current_token = push_token(&root, token.TokenType.Header, header_level) + push_text(current_token, text) + current_token = &root + continue + } + + if id, ftext, ok := parse_footnote_def(line); ok { + current_token = push_token(&root, token.TokenType.FootnoteDef, id) + push_text(current_token, ftext) + current_token = &root + continue + } + + if strings.has_prefix(line, "> ") { + if current_token.type != token.TokenType.Blockquote { + current_token = push_token(&root, token.TokenType.Blockquote, nil) + push_token(current_token, token.TokenType.Paragraph, nil) + } + last_p := current_token.children[len(current_token.children) - 1] + push_text_line(last_p, line[2:]) + continue + } + + if strings.has_prefix(line, ". ") || strings.has_prefix(line, "- ") { + if current_token.type != token.TokenType.UnorderedList { + current_token = push_token(&root, token.TokenType.UnorderedList, nil) + } + li := push_token(current_token, token.TokenType.ListItem, nil) + push_text(li, line[2:]) + continue + } + + if len(line) >= 3 { + item := 1 + idx := 0 + for idx < len(line) && line[idx] >= '0' && line[idx] <= '9' { + idx += 1 + } + + if idx > 0 && idx + 1 < len(line) && line[idx] == '.' && line[idx + 1] == ' ' { + item, _ = strconv.parse_int(line[:idx]) + remaining_text := line[idx + 2:] + + if current_token.type != token.TokenType.OrderedList { + current_token = push_token(&root, token.TokenType.OrderedList, item) + } + li := push_token(current_token, token.TokenType.ListItem, nil) + push_text(li, remaining_text) + continue + } + } + + if len(line) >= 3 && line[0] == '|' && line[len(line) - 1] == '|' { + if current_token.type != token.TokenType.Table { + current_token = push_token(&root, token.TokenType.Table, nil) + } + + if is_table_separator(line) { + row_count := len(current_token.children) + if row_count == 1 { + current_token.children[0].type = token.TokenType.TableHeaderRow + } else if row_count > 1 { + push_token(current_token, token.TokenType.TableSeparator, nil) + } + } else { + row := push_token(current_token, token.TokenType.TableRow, nil) + cells := parse_table_cells(line) + for cell_text in cells { + cell := push_token(row, token.TokenType.TableCell, nil) + push_text(cell, cell_text) + } + } + continue + } + + if current_token.type == .Blockquote { + last_p := current_token.children[len(current_token.children) - 1] + push_text_line(last_p, line) + } else if current_token.type == .UnorderedList || current_token.type == .OrderedList { + last_li := current_token.children[len(current_token.children) - 1] + push_text_line(last_li, line) + } else { + if current_token.type != .Paragraph { + current_token = push_token(&root, token.TokenType.Paragraph, nil) + } + push_text_line(current_token, line) + } + } + + for tok in root.children { + parse_inline_code(tok) + parse_inline_footnotes(tok) + parse_links(tok) + parse_variables(tok) + parse_embeds(tok) + pass_through_children(tok) + decode_text_escapes(tok) + } + + vars: map[string]string + for child in root.children { + resolve_variables(child, &vars) + } + + footnotes: [dynamic]^token.Token + new_root_children: [dynamic]^token.Token + for child in root.children { + if child.type == .FootnoteDef { + append(&footnotes, child) + } else { + append(&new_root_children, child) + } + } + for child in footnotes { + append(&new_root_children, child) + } + root.children = new_root_children + + return root +} diff --git a/src/parser/table.odin b/src/parser/table.odin new file mode 100644 index 0000000..e8b2f8b --- /dev/null +++ b/src/parser/table.odin @@ -0,0 +1,34 @@ +package parser + +import "../token" +import "core:strings" + +is_table_separator :: proc(line: string) -> bool { + inner := line[1:len(line) - 1] + cells := strings.split(inner, "|", context.temp_allocator) + if len(cells) == 0 { + return false + } + for cell in cells { + trimmed := strings.trim_space(cell) + if len(trimmed) == 0 { + return false + } + for ch in trimmed { + if ch != '-' { + return false + } + } + } + return true +} + +parse_table_cells :: proc(line: string) -> []string { + inner := line[1:len(line) - 1] + cells := strings.split(inner, "|", context.temp_allocator) + result := make([]string, len(cells), context.temp_allocator) + for cell, i in cells { + result[i] = strings.trim_space(cell) + } + return result +} diff --git a/src/parser/utils.odin b/src/parser/utils.odin new file mode 100644 index 0000000..af058ec --- /dev/null +++ b/src/parser/utils.odin @@ -0,0 +1,106 @@ +package parser + +import "../token" +import "core:strings" + +push_token :: proc( + parent: ^token.Token, + type: token.TokenType, + value: token.TokenValue = nil, +) -> ^token.Token { + tok := new(token.Token) + tok^ = token.Token{type, value, nil, parent} + append(&parent.children, tok) + return tok +} + +is_space_byte :: proc(b: byte) -> bool { + return b == ' ' || b == '\t' +} + +parse_header :: proc(line: string) -> (int, string) { + if len(line) == 0 { + return 0, line + } + + lvl := 0 + for lvl < len(line) && line[lvl] == '!' { + lvl += 1 + } + if lvl == 0 || lvl >= len(line) || line[lvl] != ' ' { + return 0, line + } + return lvl, line[lvl + 1:] +} + +parse_footnote_def :: proc(line: string) -> (string, string, bool) { + if strings.has_prefix(line, "[&&") { + idx := strings.index_byte(line, ']') + if idx != -1 { + id := line[3:idx] + text := strings.trim_space(line[idx + 1:]) + return id, text, true + } + } + return "", "", false +} + +push_text :: proc(parent: ^token.Token, text: string) { + align_type, has_align := get_alignment_token(text) + cleaned := strip_alignment_markers(text) + + target := parent + if has_align { + target = push_token(parent, align_type, nil) + } + push_token(target, token.TokenType.Text, cleaned) +} + +push_text_line :: proc(target: ^token.Token, text: string) { + if len(target.children) > 0 { + br := new(token.Token) + br^ = token.Token{token.TokenType.BreakLine, nil, nil, target} + append(&target.children, br) + } + push_text(target, text) +} + +get_alignment_token :: proc(text: string) -> (token.TokenType, bool) { + trimmed := strings.trim_space(text) + if len(trimmed) == 0 { + return .Text, false + } + if strings.has_suffix(trimmed, ">>") || strings.has_prefix(trimmed, ">>") { + return .Right, true + } + if strings.has_suffix(trimmed, "<<") || strings.has_prefix(trimmed, "<<") { + return .Left, true + } + if strings.has_suffix(trimmed, "^^") || strings.has_prefix(trimmed, "^^") { + return .Middle, true + } + return .Text, false +} + +strip_alignment_markers :: proc(text: string) -> string { + trimmed := strings.trim_space(text) + if strings.has_suffix(trimmed, ">>") { + return strings.trim_space(trimmed[:len(trimmed) - 2]) + } + if strings.has_prefix(trimmed, ">>") { + return strings.trim_space(trimmed[2:]) + } + if strings.has_suffix(trimmed, "<<") { + return strings.trim_space(trimmed[:len(trimmed) - 2]) + } + if strings.has_prefix(trimmed, "<<") { + return strings.trim_space(trimmed[2:]) + } + if strings.has_suffix(trimmed, "^^") { + return strings.trim_space(trimmed[:len(trimmed) - 2]) + } + if strings.has_prefix(trimmed, "^^") { + return strings.trim_space(trimmed[2:]) + } + return trimmed +} diff --git a/src/renderer/escape.odin b/src/renderer/escape.odin new file mode 100644 index 0000000..3b9b0c7 --- /dev/null +++ b/src/renderer/escape.odin @@ -0,0 +1,25 @@ +package renderer + +import "../parser" +import "core:strings" + +escape_html :: proc(s: string, allocator := context.temp_allocator) -> string { + escaped := parser.decode_escapes(s, allocator) + + b := strings.builder_make(allocator) + for ch in escaped { + switch ch { + case '&': + strings.write_string(&b, "&") + case '<': + strings.write_string(&b, "<") + case '>': + strings.write_string(&b, ">") + case '"': + strings.write_string(&b, """) + case: + strings.write_rune(&b, ch) + } + } + return strings.to_string(b) +} diff --git a/src/renderer/html.odin b/src/renderer/html.odin new file mode 100644 index 0000000..9a9d646 --- /dev/null +++ b/src/renderer/html.odin @@ -0,0 +1,330 @@ +package renderer + +import "../mime" +import "../token" +import "core:fmt" +import "core:strings" + +render_aligned_cell :: proc(tok: ^token.Token, tag: string) { + align := get_alignment_attribute(tok) + if align != "" { + fmt.printf("<%s align=\"%s\">", tag, align) + for child in tok.children[0].children { + print_html(child) + } + } else { + fmt.printf("<%s>", tag) + for child in tok.children { + print_html(child) + } + } + fmt.printf("", tag) +} + +render_simple_wrapper :: proc(tok: ^token.Token, tag: string) { + fmt.printf("<%s>", tag) + for child in tok.children { + print_html(child) + } + fmt.printf("", tag) +} + +render_div_wrapper :: proc(tok: ^token.Token, align: string) { + fmt.printf("
    ", align) + for child in tok.children { + print_html(child) + } + fmt.print("
    ") +} + +render_container :: proc(tok: ^token.Token, tag: string) { + fmt.printf("<%s>\n", tag) + for child in tok.children { + print_html(child) + } + fmt.printf("\n", tag) +} + +render_media_object :: proc(tok: ^token.Token, mime_type: string) { + url, ok := tok.value.(string) + if !ok { + return + } + b := strings.builder_make(context.temp_allocator) + for child in tok.children { + get_text_content(child, &b) + } + alt_text := strings.to_string(b) + if alt_text == "" { + alt_text = url + } + fmt.printf("\n", escape_html(url), escape_html(mime_type)) + fmt.printf("\n", escape_html(url)) + fmt.printf("\n") + fmt.printf("\n") + fmt.printf("%s\n", escape_html(url), escape_html(alt_text)) + fmt.print("\n") +} + +print_html :: proc(tok: ^token.Token) { + switch tok.type { + case token.TokenType.Root: + has_footnotes := false + for child in tok.children { + if child.type == .FootnoteDef && !has_footnotes { + fmt.print("
    \n") + has_footnotes = true + } + print_html(child) + } + + case token.TokenType.Header: + level := 1 + if v, ok := tok.value.(int); ok { + level = v + } + + b := strings.builder_make(context.temp_allocator) + for child in tok.children { + get_text_content(child, &b) + } + slug := slugifyifyifyify(strings.to_string(b)) + + fmt.printf("", level, escape_html(slug)) + for child in tok.children { + print_html(child) + } + fmt.printf("\n", level) + + case token.TokenType.Paragraph: + only_var_defs := check_only_var_defs(tok) + if !only_var_defs { + fmt.print("

    ") + for child in tok.children { + print_html(child) + } + fmt.print("

    \n") + } else { + for child in tok.children { + if child.type != .BreakLine { + print_html(child) + } + } + } + + case token.TokenType.Blockquote: + render_container(tok, "blockquote") + + case token.TokenType.UnorderedList: + render_container(tok, "ul") + + case token.TokenType.OrderedList: + render_container(tok, "ol") + + case token.TokenType.ListItem: + fmt.print("
  • ") + for child in tok.children { + print_html(child) + } + fmt.print("
  • \n") + + case token.TokenType.Bold: + render_simple_wrapper(tok, "strong") + + case token.TokenType.Italics: + render_simple_wrapper(tok, "em") + + case token.TokenType.Strikethrough: + render_simple_wrapper(tok, "strike") + + case token.TokenType.Table: + render_container(tok, "table") + + case token.TokenType.TableHeaderRow: + fmt.print("") + for child in tok.children { + render_aligned_cell(child, "th") + } + fmt.print("\n") + + case token.TokenType.TableRow: + fmt.print("") + for child in tok.children { + print_html(child) + } + fmt.print("\n") + + case token.TokenType.TableCell: + render_aligned_cell(tok, "td") + + case token.TokenType.TableSeparator: + fmt.print("\n") + + case token.TokenType.Left: + render_div_wrapper(tok, "left") + + case token.TokenType.Right: + render_div_wrapper(tok, "right") + + case token.TokenType.Middle: + render_div_wrapper(tok, "center") + + case token.TokenType.FootnoteRef: + if id, ok := tok.value.(string); ok { + fmt.printf( + "%s", + escape_html(id), + escape_html(id), + escape_html(id), + ) + } + + case token.TokenType.FootnoteDef: + if id, ok := tok.value.(string); ok { + fmt.printf( + "
    ^ ", + escape_html(id), + escape_html(id), + ) + for child in tok.children { + print_html(child) + } + fmt.print("
    \n") + } + + case token.TokenType.CodeBlock: + lang := "" + if v, ok := tok.value.(string); ok && v != "" { + lang = v + } + if lang != "" { + fmt.printf("
    ", lang)
    +		} else {
    +			fmt.print("
    ")
    +		}
    +		for i := 0; i < len(tok.children); i += 1 {
    +			print_html(tok.children[i])
    +			if i + 1 < len(tok.children) {
    +				fmt.print("\n")
    +			}
    +		}
    +		fmt.print("\n
    \n") + + case token.TokenType.InlineCode: + if code, ok := tok.value.(string); ok { + fmt.printf("%s", escape_html(code)) + } + + case token.TokenType.HorizontalRule: + fmt.print("
    \n") + + case token.TokenType.Underline: + fmt.print("") + for child in tok.children { + print_html(child) + } + fmt.print("") + + case token.TokenType.Superscript: + fmt.print("") + for child in tok.children { + print_html(child) + } + fmt.print("") + + case token.TokenType.Subscript: + fmt.print("") + for child in tok.children { + print_html(child) + } + fmt.print("") + + case token.TokenType.HeaderLink: + if target, ok := tok.value.(string); ok { + slug := slugifyifyifyify(target) + fmt.printf("", escape_html(slug)) + for child in tok.children { + print_html(child) + } + fmt.print("") + } + + case token.TokenType.Link: + if url, ok := tok.value.(string); ok { + fmt.printf("", escape_html(url)) + for child in tok.children { + print_html(child) + } + fmt.print("") + } + + case token.TokenType.Image: + if url, ok := tok.value.(string); ok { + b := strings.builder_make(context.temp_allocator) + for child in tok.children { + get_text_content(child, &b) + } + alt_text := strings.to_string(b) + fmt.printf("\"%s\"", escape_html(url), escape_html(alt_text)) + } + + case token.TokenType.Audio: + if url, ok := tok.value.(string); ok { + render_media_object(tok, mime.detect_mime_type(url)) + } + + case token.TokenType.Video: + if url, ok := tok.value.(string); ok { + render_media_object(tok, mime.detect_mime_type(url)) + } + + case token.TokenType.Iframe: + if url, ok := tok.value.(string); ok { + fmt.printf("", escape_html(url)) + } + + case token.TokenType.Script: + if url, ok := tok.value.(string); ok { + fmt.printf("", escape_html(url)) + } + + case token.TokenType.ScriptBlock: + if content, ok := tok.value.(string); ok { + fmt.printf("", content) + } + + case token.TokenType.Variable: + if name, ok := tok.value.(string); ok { + fmt.printf("", escape_html(name)) + for child in tok.children { + print_html(child) + } + fmt.print("") + } + + case token.TokenType.VariableDef: + if name, ok := tok.value.(string); ok { + b := strings.builder_make(context.temp_allocator) + for child in tok.children { + get_text_content(child, &b) + } + val := strings.to_string(b) + fmt.printf( + "", + escape_html(name), + escape_html(val), + ) + } + + case token.TokenType.BreakLine: + fmt.print("
    \n") + + case token.TokenType.Text: + if v, ok := tok.value.(string); ok { + fmt.print(escape_html(v)) + } + for child in tok.children { + print_html(child) + } + } +} diff --git a/src/renderer/utils.odin b/src/renderer/utils.odin new file mode 100644 index 0000000..e4117a9 --- /dev/null +++ b/src/renderer/utils.odin @@ -0,0 +1,87 @@ +package renderer + +import "../token" +import "core:fmt" +import "core:strings" +import "core:unicode" + +get_text_content :: proc(token: ^token.Token, b: ^strings.Builder) { + if token.type == .Text { + if v, ok := token.value.(string); ok { + strings.write_string(b, v) + } + } + for child in token.children { + get_text_content(child, b) + } +} + +check_only_var_defs :: proc(token: ^token.Token) -> bool { + for child in token.children { + if child.type == .VariableDef || child.type == .BreakLine { + continue + } + if child.type == .Text { + if child.value != nil { + if v, ok := child.value.(string); ok && v != "" { + return false + } + } + has_non_var_def := false + for gc in child.children { + if gc.type != .VariableDef { + has_non_var_def = true + break + } + } + if has_non_var_def { + return false + } + continue + } + return false + } + return true +} + +slugifyifyifyify :: proc(text: string) -> string { + b := strings.builder_make(context.temp_allocator) + + for ch in text { + lower_ch := unicode.to_lower(ch) + + if unicode.is_letter(lower_ch) || unicode.is_digit(lower_ch) { + strings.write_rune(&b, lower_ch) + } else if lower_ch == ' ' || lower_ch == '_' || lower_ch == '-' { + strings.write_rune(&b, '-') + } + } + + res := strings.to_string(b) + + for { + new_res, replaced := strings.replace_all(res, "--", "-", context.temp_allocator) + if !replaced { + break + } + res = new_res + } + + res = strings.trim(res, "-") + + return res +} + +get_alignment_attribute :: proc(tok: ^token.Token) -> string { + if len(tok.children) > 0 { + first_child := tok.children[0] + if first_child.type == .Left { + return "left" + } else if first_child.type == .Right { + return "right" + } else if first_child.type == .Middle { + return "center" + } + } + return "" +} diff --git a/src/token/token.odin b/src/token/token.odin new file mode 100644 index 0000000..dfe216d --- /dev/null +++ b/src/token/token.odin @@ -0,0 +1,54 @@ +package token + +TokenType :: enum { + Text, + Italics, + Bold, + BreakLine, + Header, + Paragraph, + Root, + Blockquote, + UnorderedList, + OrderedList, + ListItem, + Strikethrough, + Table, + TableRow, + TableHeaderRow, + TableCell, + TableSeparator, + Left, + Right, + Middle, + FootnoteDef, + FootnoteRef, + CodeBlock, + InlineCode, + HorizontalRule, + Underline, + Superscript, + Subscript, + HeaderLink, + Link, + Image, + Audio, + Video, + Iframe, + Script, + ScriptBlock, + Variable, + VariableDef, +} + +TokenValue :: union { + string, + int, +} + +Token :: struct { + type: TokenType, + value: TokenValue, + children: [dynamic]^Token, + parent: ^Token, +}