[OCaml] Org mode parser.

refactor lexers produce parser tokens directly

Eliminate the token conversion boilerplate in parse_bridge.ml. Lexer now returns Parser.token values and Inline_lexer returns Inline_parser.token values directly. - Extract list_item_data into Token module (breaks the circular dependency between Lexer and Parser) - Remove Lexer.token and Inline_lexer.token type definitions - parse_bridge.ml shrinks to just the Menhir wiring glue

Commit
5633ebe0058d7b20730277b645c07f690443c8be
Author
Kiro (high) <noreply@kiro.dev>
Author date
Committer
Marius Peter <dev@marius-peter.com>
Committer date
Changed files
lib/inline_lexer.ml
index 6148d84f..5ed66ca1 100644..100644
@@ -5,30 +5,8 @@
5 5
6 6 Key design: emphasis delimiters are only emitted as OPEN/CLOSE tokens when
7 7 they form valid matched pairs. Unmatched delimiters become TEXT. This
8 Removed: eliminates ambiguity in the Menhir grammar. *)
9 Removed:
10 Removed: (** Inline tokens. *)
11 Removed: type token =
12 Removed: | TEXT of string
13 Removed: | STAR_OPEN
14 Removed: | STAR_CLOSE
15 Removed: | SLASH_OPEN
16 Removed: | SLASH_CLOSE
17 Removed: | UNDER_OPEN
18 Removed: | UNDER_CLOSE
19 Removed: | PLUS_OPEN
20 Removed: | PLUS_CLOSE
21 Removed: | TILDE of string
22 Removed: | EQUALS of string
23 Removed: | LINK_OPEN
24 Removed: | LINK_CLOSE
25 Removed: | LINK_SEP
26 Removed: | TIMESTAMP of string
27 Removed: | ANGLE_LINK of string
28 Removed: | PLAIN_LINK of string
29 Removed: | LINE_BREAK
30 Removed: | NEWLINE
31 Removed: | EOF
8 Added: eliminates ambiguity in the Menhir grammar. Tokens are [Inline_parser.token]
9 Added: values directly. *)
32 10
33 11 (** PRE characters that allow an emphasis marker to open. *)
34 12 let is_pre ch =
@@ -250,48 +228,48 @@
250 228
251 229 (** Get the next token. *)
252 230 let next_token st =
253 Removed: if st.pos >= st.len then EOF
231 Added: if st.pos >= st.len then Inline_parser.EOF
254 232 else
255 233 let c = st.input.[st.pos] in
256 234 (* Try line break *)
257 235 match try_line_break st with
258 236 | Some new_pos ->
259 237 st.pos <- new_pos;
260 Removed: LINE_BREAK
238 Added: Inline_parser.LINE_BREAK
261 239 | None ->
262 240 (* Try verbatim/code *)
263 241 if c = '~' then (
264 242 match try_verbatim_code st '~' with
265 243 | Some (content, new_pos) ->
266 244 st.pos <- new_pos;
267 Removed: TILDE content
245 Added: Inline_parser.TILDE content
268 246 | None ->
269 247 st.pos <- st.pos + 1;
270 Removed: TEXT "~")
248 Added: Inline_parser.TEXT "~")
271 249 else if c = '=' then (
272 250 match try_verbatim_code st '=' with
273 251 | Some (content, new_pos) ->
274 252 st.pos <- new_pos;
275 Removed: EQUALS content
253 Added: Inline_parser.EQUALS content
276 254 | None ->
277 255 st.pos <- st.pos + 1;
278 Removed: TEXT "=")
256 Added: Inline_parser.TEXT "=")
279 257 else if
280 258 (* Try link brackets *)
281 259 c = '[' && st.pos + 1 < st.len && st.input.[st.pos + 1] = '['
282 260 then begin
283 261 st.pos <- st.pos + 2;
284 Removed: LINK_OPEN
262 Added: Inline_parser.LINK_OPEN
285 263 end
286 264 else if c = ']' && st.pos + 1 < st.len && st.input.[st.pos + 1] = ']'
287 265 then begin
288 266 st.pos <- st.pos + 2;
289 Removed: LINK_CLOSE
267 Added: Inline_parser.LINK_CLOSE
290 268 end
291 269 else if c = ']' && st.pos + 1 < st.len && st.input.[st.pos + 1] = '['
292 270 then begin
293 271 st.pos <- st.pos + 2;
294 Removed: LINK_SEP
272 Added: Inline_parser.LINK_SEP
295 273 end
296 274 else if
297 275 (* Try timestamp *)
@@ -304,19 +282,19 @@
304 282 begin match try_timestamp st with
305 283 | Some (ts, new_pos) ->
306 284 st.pos <- new_pos;
307 Removed: TIMESTAMP ts
285 Added: Inline_parser.TIMESTAMP ts
308 286 | None ->
309 287 if c = '<' then (
310 288 match try_angle_link st with
311 289 | Some (content, new_pos) ->
312 290 st.pos <- new_pos;
313 Removed: ANGLE_LINK content
291 Added: Inline_parser.ANGLE_LINK content
314 292 | None ->
315 293 st.pos <- st.pos + 1;
316 Removed: TEXT "<")
294 Added: Inline_parser.TEXT "<")
317 295 else begin
318 296 st.pos <- st.pos + 1;
319 Removed: TEXT "["
297 Added: Inline_parser.TEXT "["
320 298 end
321 299 end
322 300 else
@@ -324,15 +302,15 @@
324 302 begin match try_plain_link st with
325 303 | Some (link, new_pos) ->
326 304 st.pos <- new_pos;
327 Removed: PLAIN_LINK link
305 Added: Inline_parser.PLAIN_LINK link
328 306 | None ->
329 307 (* Try emphasis markers — only emit OPEN/CLOSE if pre-scan found a pair *)
330 308 if c = '*' then begin
331 309 let tok =
332 310 match Hashtbl.find_opt st.star_pairs st.pos with
333 Removed: | Some `Open -> STAR_OPEN
334 Removed: | Some `Close -> STAR_CLOSE
335 Removed: | None -> TEXT "*"
311 Added: | Some `Open -> Inline_parser.STAR_OPEN
312 Added: | Some `Close -> Inline_parser.STAR_CLOSE
313 Added: | None -> Inline_parser.TEXT "*"
336 314 in
337 315 st.pos <- st.pos + 1;
338 316 tok
@@ -340,9 +318,9 @@
340 318 else if c = '/' then begin
341 319 let tok =
342 320 match Hashtbl.find_opt st.slash_pairs st.pos with
343 Removed: | Some `Open -> SLASH_OPEN
344 Removed: | Some `Close -> SLASH_CLOSE
345 Removed: | None -> TEXT "/"
321 Added: | Some `Open -> Inline_parser.SLASH_OPEN
322 Added: | Some `Close -> Inline_parser.SLASH_CLOSE
323 Added: | None -> Inline_parser.TEXT "/"
346 324 in
347 325 st.pos <- st.pos + 1;
348 326 tok
@@ -350,9 +328,9 @@
350 328 else if c = '_' then begin
351 329 let tok =
352 330 match Hashtbl.find_opt st.under_pairs st.pos with
353 Removed: | Some `Open -> UNDER_OPEN
354 Removed: | Some `Close -> UNDER_CLOSE
355 Removed: | None -> TEXT "_"
331 Added: | Some `Open -> Inline_parser.UNDER_OPEN
332 Added: | Some `Close -> Inline_parser.UNDER_CLOSE
333 Added: | None -> Inline_parser.TEXT "_"
356 334 in
357 335 st.pos <- st.pos + 1;
358 336 tok
@@ -360,16 +338,16 @@
360 338 else if c = '+' then begin
361 339 let tok =
362 340 match Hashtbl.find_opt st.plus_pairs st.pos with
363 Removed: | Some `Open -> PLUS_OPEN
364 Removed: | Some `Close -> PLUS_CLOSE
365 Removed: | None -> TEXT "+"
341 Added: | Some `Open -> Inline_parser.PLUS_OPEN
342 Added: | Some `Close -> Inline_parser.PLUS_CLOSE
343 Added: | None -> Inline_parser.TEXT "+"
366 344 in
367 345 st.pos <- st.pos + 1;
368 346 tok
369 347 end
370 348 else if c = '\n' then begin
371 349 st.pos <- st.pos + 1;
372 Removed: NEWLINE
350 Added: Inline_parser.NEWLINE
373 351 end
374 352 else begin
375 353 (* Accumulate plain text *)
@@ -391,7 +369,7 @@
391 369 in
392 370 let end_pos = scan (start + 1) in
393 371 st.pos <- end_pos;
394 Removed: TEXT (String.sub st.input start (end_pos - start))
372 Added: Inline_parser.TEXT (String.sub st.input start (end_pos - start))
395 373 end
396 374 end
397 375
@@ -400,6 +378,8 @@
400 378 let st = create input in
401 379 let rec loop acc =
402 380 let tok = next_token st in
403 Removed: match tok with EOF -> List.rev (EOF :: acc) | _ -> loop (tok :: acc)
381 Added: match tok with
382 Added: | Inline_parser.EOF -> List.rev (Inline_parser.EOF :: acc)
383 Added: | _ -> loop (tok :: acc)
404 384 in
405 385 loop []
lib/lexer.ml
index d543a5da..53b34b2d 100644..100644
@@ -1,4 +1,4 @@
1 Removed: (** Handwritten structural line lexer.
1 Added: (** "Handwritten" structural line lexer.
2 2
3 3 Classifies physical lines into high-level structural tokens that the Menhir
4 4 grammar will consume. The lexer is context-aware: it knows the TODO keywords
@@ -6,37 +6,8 @@
6 6 parsing is deferred to normalization).
7 7
8 8 The lexer operates line-by-line. Each call to [next_token] returns the
9 Removed: classification of the next line. *)
10 Removed:
11 Removed: type list_item_data = {
12 Removed: lid_indent : int;
13 Removed: lid_bullet : string; (** The bullet text including trailing space *)
14 Removed: lid_counter_set : int option;
15 Removed: lid_checkbox : string option; (** "[ ]", "[X]", "[-]" *)
16 Removed: lid_rest : string; (** Remainder of line after bullet/checkbox *)
17 Removed: }
18 Removed: (** A raw list item token payload. *)
19 Removed:
20 Removed: (** Structural tokens produced by the lexer. *)
21 Removed: type token =
22 Removed: | Blank
23 Removed: | Heading of int * string (** level, rest of line *)
24 Removed: | Keyword of string * string (** key (uppercase), value *)
25 Removed: | Affiliated_keyword of string * string option * string
26 Removed: (** name, optional, value *)
27 Removed: | Begin_block of string * string option (** name (lowercase), params *)
28 Removed: | End_block of string (** name (lowercase) *)
29 Removed: | Drawer_begin of string (** name *)
30 Removed: | Drawer_end
31 Removed: | Property of string * string option (** name, value *)
32 Removed: | Planning of string (** raw planning line content *)
33 Removed: | List_item of list_item_data
34 Removed: | Table_row of string (** raw row content including leading | *)
35 Removed: | Fixed_width of string (** content after ": " *)
36 Removed: | Comment_line of string (** content after "# " *)
37 Removed: | Horizontal_rule
38 Removed: | Text_line of string (** anything else *)
39 Removed: | Eof
9 Added: classification of the next line. Tokens are [Parser.token] values directly.
10 Added: *)
40 11
41 12 type state = {
42 13 lines : string array;
@@ -371,7 +342,7 @@
371 342 in
372 343 Some
373 344 {
374 Removed: lid_indent = indent;
345 Added: Token.lid_indent = indent;
375 346 lid_bullet = bullet;
376 347 lid_counter_set = counter_set;
377 348 lid_checkbox = checkbox;
@@ -442,94 +413,95 @@
442 413 (** Classify the next line and return its token. *)
443 414 let classify_line _st line =
444 415 let line = rstrip line in
445 Removed: if is_blank_line line then Blank
416 Added: if is_blank_line line then Parser.BLANK
446 417 else
447 418 let indent = leading_indent line in
448 419 let start = indent in
449 420 (* Headings must be at column 0 *)
450 421 if indent = 0 then
451 422 begin match try_heading line with
452 Removed: | Some (level, rest) -> Heading (level, rest)
423 Added: | Some (level, rest) -> Parser.HEADING (level, rest)
453 424 | None -> (
454 425 (* Try keyword/block *)
455 426 match try_begin_block line start with
456 Removed: | Some (name, params) -> Begin_block (name, params)
427 Added: | Some (name, params) -> Parser.BEGIN_BLOCK (name, params)
457 428 | None -> (
458 429 match try_end_block line start with
459 Removed: | Some name -> End_block name
430 Added: | Some name -> Parser.END_BLOCK name
460 431 | None -> (
461 432 match try_keyword line start with
462 433 | Some (key, value) ->
463 434 if is_affiliated_key key then
464 Removed: (* Check for optional value: #+KEY[OPTVAL]: VALUE *)
465 Removed: Affiliated_keyword
435 Added: Parser.AFFILIATED_KEYWORD
466 436 (String.uppercase_ascii key, None, value)
467 Removed: else Keyword (String.uppercase_ascii key, value)
437 Added: else Parser.KEYWORD (String.uppercase_ascii key, value)
468 438 | None -> (
469 Removed: if is_drawer_end line start then Drawer_end
439 Added: if is_drawer_end line start then Parser.DRAWER_END
470 440 else
471 441 match try_drawer_begin line start with
472 Removed: | Some name -> Drawer_begin name
442 Added: | Some name -> Parser.DRAWER_BEGIN name
473 443 | None -> (
474 444 match try_property line start with
475 Removed: | Some (name, value) -> Property (name, value)
445 Added: | Some (name, value) -> Parser.PROPERTY (name, value)
476 446 | None -> (
477 447 if is_horizontal_rule line start then
478 Removed: Horizontal_rule
448 Added: Parser.HORIZONTAL_RULE
479 449 else if is_planning_line line start then
480 Removed: Planning line
450 Added: Parser.PLANNING line
481 451 else if is_table_row line start then
482 Removed: Table_row line
452 Added: Parser.TABLE_ROW line
483 453 else
484 454 match try_comment_line line start with
485 Removed: | Some content -> Comment_line content
455 Added: | Some content -> Parser.COMMENT_LINE content
486 456 | None -> (
487 457 match try_fixed_width line start with
488 Removed: | Some content -> Fixed_width content
458 Added: | Some content ->
459 Added: Parser.FIXED_WIDTH content
489 460 | None -> (
490 461 match try_list_item line with
491 Removed: | Some data -> List_item data
492 Removed: | None -> Text_line line))))))))
462 Added: | Some data -> Parser.LIST_ITEM data
463 Added: | None -> Parser.TEXT_LINE line)))))))
464 Added: )
493 465 end
494 466 else
495 467 (* Indented lines *)
496 Removed: begin if is_planning_line line start then Planning line
497 Removed: else if is_table_row line start then Table_row line
498 Removed: else if is_drawer_end line start then Drawer_end
468 Added: begin if is_planning_line line start then Parser.PLANNING line
469 Added: else if is_table_row line start then Parser.TABLE_ROW line
470 Added: else if is_drawer_end line start then Parser.DRAWER_END
499 471 else
500 472 match try_drawer_begin line start with
501 Removed: | Some name -> Drawer_begin name
473 Added: | Some name -> Parser.DRAWER_BEGIN name
502 474 | None -> (
503 475 match try_begin_block line start with
504 Removed: | Some (name, params) -> Begin_block (name, params)
476 Added: | Some (name, params) -> Parser.BEGIN_BLOCK (name, params)
505 477 | None -> (
506 478 match try_end_block line start with
507 Removed: | Some name -> End_block name
479 Added: | Some name -> Parser.END_BLOCK name
508 480 | None -> (
509 481 match try_keyword line start with
510 482 | Some (key, value) ->
511 483 if is_affiliated_key key then
512 Removed: Affiliated_keyword
484 Added: Parser.AFFILIATED_KEYWORD
513 485 (String.uppercase_ascii key, None, value)
514 Removed: else Keyword (String.uppercase_ascii key, value)
486 Added: else Parser.KEYWORD (String.uppercase_ascii key, value)
515 487 | None -> (
516 488 match try_property line start with
517 Removed: | Some (name, value) -> Property (name, value)
489 Added: | Some (name, value) -> Parser.PROPERTY (name, value)
518 490 | None -> (
519 491 match try_comment_line line start with
520 Removed: | Some content -> Comment_line content
492 Added: | Some content -> Parser.COMMENT_LINE content
521 493 | None -> (
522 494 match try_fixed_width line start with
523 Removed: | Some content -> Fixed_width content
495 Added: | Some content -> Parser.FIXED_WIDTH content
524 496 | None -> (
525 497 match try_list_item line with
526 Removed: | Some data -> List_item data
527 Removed: | None -> Text_line line)))))))
498 Added: | Some data -> Parser.LIST_ITEM data
499 Added: | None -> Parser.TEXT_LINE line)))))))
528 500 end
529 501
530 502 (** Get the next token from the lexer. *)
531 503 let next_token st =
532 Removed: if st.line_idx >= Array.length st.lines then Eof
504 Added: if st.line_idx >= Array.length st.lines then Parser.EOF
533 505 else begin
534 506 let line = st.lines.(st.line_idx) in
535 507 st.line_idx <- st.line_idx + 1;
@@ -545,7 +517,7 @@
545 517 let ln = st.line_number in
546 518 let tok = next_token st in
547 519 match tok with
548 Removed: | Eof -> List.rev ((Eof, ln) :: acc)
520 Added: | Parser.EOF -> List.rev ((Parser.EOF, ln) :: acc)
549 521 | _ -> loop ((tok, ln) :: acc)
550 522 in
551 523 loop []
lib/org.ml
index c0e72f8b..03f6344b 100644..100644
@@ -22,6 +22,8 @@
22 22
23 23 module Private = struct
24 24 module Lexer = Lexer
25 Added: module Parser = Parser
26 Added: module Token = Token
25 27 module Parse_bridge = Parse_bridge
26 28 module Raw_ast = Raw_ast
27 29 module Prescan = Prescan
lib/parse_bridge.ml
index 137b0020..20bd3351 100644..100644
@@ -1,29 +1,8 @@
1 Removed: (** Bridge between the structural lexer and the Menhir parser.
2 Removed:
3 Removed: Converts [Lexer.token] values into [Parser.token] values and provides the
4 Removed: entry point for structural parsing. *)
1 Added: (** Bridge between the handwritten lexers and the Menhir parsers.
5 2
6 Removed: (** Convert a lexer token to a Menhir parser token. *)
7 Removed: let convert_token (tok : Lexer.token) : Parser.token =
8 Removed: match tok with
9 Removed: | Lexer.Blank -> Parser.BLANK
10 Removed: | Lexer.Heading (level, rest) -> Parser.HEADING (level, rest)
11 Removed: | Lexer.Keyword (key, value) -> Parser.KEYWORD (key, value)
12 Removed: | Lexer.Affiliated_keyword (name, opt, value) ->
13 Removed: Parser.AFFILIATED_KEYWORD (name, opt, value)
14 Removed: | Lexer.Begin_block (name, params) -> Parser.BEGIN_BLOCK (name, params)
15 Removed: | Lexer.End_block name -> Parser.END_BLOCK name
16 Removed: | Lexer.Drawer_begin name -> Parser.DRAWER_BEGIN name
17 Removed: | Lexer.Drawer_end -> Parser.DRAWER_END
18 Removed: | Lexer.Property (name, value) -> Parser.PROPERTY (name, value)
19 Removed: | Lexer.Planning line -> Parser.PLANNING line
20 Removed: | Lexer.List_item data -> Parser.LIST_ITEM data
21 Removed: | Lexer.Table_row content -> Parser.TABLE_ROW content
22 Removed: | Lexer.Fixed_width content -> Parser.FIXED_WIDTH content
23 Removed: | Lexer.Comment_line content -> Parser.COMMENT_LINE content
24 Removed: | Lexer.Horizontal_rule -> Parser.HORIZONTAL_RULE
25 Removed: | Lexer.Text_line content -> Parser.TEXT_LINE content
26 Removed: | Lexer.Eof -> Parser.EOF
3 Added: Provides entry points that wire lexer state into the Menhir-generated parser
4 Added: functions. No token conversion is needed since the lexers produce parser
5 Added: tokens directly. *)
27 6
28 7 (** A dummy lexbuf position for Menhir (we track positions ourselves). *)
29 8 let dummy_pos =
@@ -38,48 +17,20 @@
38 17 dummy_lexbuf.Lexing.lex_curr_p <- dummy_pos;
39 18 let token_fun _lexbuf =
40 19 let tok = Lexer.next_token lexer_state in
41 Removed: let menhir_tok = convert_token tok in
42 20 let ln = Lexer.line_number lexer_state in
43 21 dummy_lexbuf.Lexing.lex_start_p <- { dummy_pos with Lexing.pos_lnum = ln };
44 22 dummy_lexbuf.Lexing.lex_curr_p <- { dummy_pos with Lexing.pos_lnum = ln };
45 Removed: menhir_tok
23 Added: tok
46 24 in
47 25 Parser.document token_fun dummy_lexbuf
48 26
49 Removed: (** Convert an inline lexer token to a Menhir inline parser token. *)
50 Removed: let convert_inline_token (tok : Inline_lexer.token) : Inline_parser.token =
51 Removed: match tok with
52 Removed: | Inline_lexer.TEXT s -> Inline_parser.TEXT s
53 Removed: | Inline_lexer.STAR_OPEN -> Inline_parser.STAR_OPEN
54 Removed: | Inline_lexer.STAR_CLOSE -> Inline_parser.STAR_CLOSE
55 Removed: | Inline_lexer.SLASH_OPEN -> Inline_parser.SLASH_OPEN
56 Removed: | Inline_lexer.SLASH_CLOSE -> Inline_parser.SLASH_CLOSE
57 Removed: | Inline_lexer.UNDER_OPEN -> Inline_parser.UNDER_OPEN
58 Removed: | Inline_lexer.UNDER_CLOSE -> Inline_parser.UNDER_CLOSE
59 Removed: | Inline_lexer.PLUS_OPEN -> Inline_parser.PLUS_OPEN
60 Removed: | Inline_lexer.PLUS_CLOSE -> Inline_parser.PLUS_CLOSE
61 Removed: | Inline_lexer.TILDE s -> Inline_parser.TILDE s
62 Removed: | Inline_lexer.EQUALS s -> Inline_parser.EQUALS s
63 Removed: | Inline_lexer.LINK_OPEN -> Inline_parser.LINK_OPEN
64 Removed: | Inline_lexer.LINK_CLOSE -> Inline_parser.LINK_CLOSE
65 Removed: | Inline_lexer.LINK_SEP -> Inline_parser.LINK_SEP
66 Removed: | Inline_lexer.TIMESTAMP s -> Inline_parser.TIMESTAMP s
67 Removed: | Inline_lexer.ANGLE_LINK s -> Inline_parser.ANGLE_LINK s
68 Removed: | Inline_lexer.PLAIN_LINK s -> Inline_parser.PLAIN_LINK s
69 Removed: | Inline_lexer.LINE_BREAK -> Inline_parser.LINE_BREAK
70 Removed: | Inline_lexer.NEWLINE -> Inline_parser.NEWLINE
71 Removed: | Inline_lexer.EOF -> Inline_parser.EOF
72 Removed:
73 27 (** Parse inline text into a list of Ast.inline objects. *)
74 28 let parse_inline input =
75 29 let st = Inline_lexer.create input in
76 30 let dummy_lexbuf = Lexing.from_string "" in
77 31 dummy_lexbuf.Lexing.lex_start_p <- dummy_pos;
78 32 dummy_lexbuf.Lexing.lex_curr_p <- dummy_pos;
79 Removed: let token_fun _lexbuf =
80 Removed: let tok = Inline_lexer.next_token st in
81 Removed: convert_inline_token tok
82 Removed: in
33 Added: let token_fun _lexbuf = Inline_lexer.next_token st in
83 34 try Inline_parser.inline_content token_fun dummy_lexbuf
84 35 with Inline_parser.Error ->
85 36 (* Fallback: if inline parsing fails, return plain text *)
lib/parser.mly
index 74b74a7c..fcb63700 100644..100644
@@ -4,13 +4,13 @@
4 4 %{
5 5 open Raw_ast
6 6
7 Removed: let convert_list_item (item : Lexer.list_item_data) : raw_list_item =
8 Removed: { rli_indent = item.Lexer.lid_indent;
9 Removed: rli_bullet = item.Lexer.lid_bullet;
10 Removed: rli_counter_set = item.Lexer.lid_counter_set;
11 Removed: rli_checkbox = item.Lexer.lid_checkbox;
7 Added: let convert_list_item (item : Token.list_item_data) : raw_list_item =
8 Added: { rli_indent = item.Token.lid_indent;
9 Added: rli_bullet = item.Token.lid_bullet;
10 Added: rli_counter_set = item.Token.lid_counter_set;
11 Added: rli_checkbox = item.Token.lid_checkbox;
12 12 rli_tag = None;
13 Removed: rli_body_lines = [item.Lexer.lid_rest];
13 Added: rli_body_lines = [item.Token.lid_rest];
14 14 }
15 15
16 16 let parse_planning_entries (line : string) : raw_planning list =
@@ -55,7 +55,7 @@
55 55 %token DRAWER_END
56 56 %token <string * string option> PROPERTY
57 57 %token <string> PLANNING
58 Removed: %token <Lexer.list_item_data> LIST_ITEM
58 Added: %token <Token.list_item_data> LIST_ITEM
59 59 %token <string> TABLE_ROW
60 60 %token <string> FIXED_WIDTH
61 61 %token <string> COMMENT_LINE
@@ -132,7 +132,7 @@
132 132 | c = COMMENT_LINE { "# " ^ c }
133 133 | f = FIXED_WIDTH { ": " ^ f }
134 134 | HORIZONTAL_RULE { "-----" }
135 Removed: | item = LIST_ITEM { item.Lexer.lid_rest }
135 Added: | item = LIST_ITEM { item.Token.lid_rest }
136 136 | row = TABLE_ROW { row }
137 137 | p = PLANNING { p }
138 138 | prop = PROPERTY { let (n, v) = prop in ":" ^ n ^ ": " ^ (Option.value ~default:"" v) }
lib/token.ml
index 00000000..6888b037 000000..100644
@@ -0,0 +1,10 @@
1 Added: (** Shared token payload types used by both lexers and parsers. *)
2 Added:
3 Added: type list_item_data = {
4 Added: lid_indent : int;
5 Added: lid_bullet : string; (** The bullet text including trailing space *)
6 Added: lid_counter_set : int option;
7 Added: lid_checkbox : string option; (** "[ ]", "[X]", "[-]" *)
8 Added: lid_rest : string; (** Remainder of line after bullet/checkbox *)
9 Added: }
10 Added: (** A raw list item token payload. *)
test/test_main.ml
index 4177aa17..c90ef4ed 100644..100644
@@ -60,33 +60,34 @@
60 60
61 61 (* Lexer tests *)
62 62 module L = Org.Private.Lexer
63 Added: module T = Org.Private.Parser
63 64
64 65 let config = Org.Config.default
65 66 let lex input = L.tokenize ~config input |> List.map fst
66 67
67 68 let tok_to_string = function
68 Removed: | L.Blank -> "Blank"
69 Removed: | L.Heading (n, s) -> Printf.sprintf "Heading(%d, %S)" n s
70 Removed: | L.Keyword (k, v) -> Printf.sprintf "Keyword(%S, %S)" k v
71 Removed: | L.Affiliated_keyword (n, _, v) -> Printf.sprintf "Affiliated(%S, %S)" n v
72 Removed: | L.Begin_block (n, p) ->
69 Added: | T.BLANK -> "Blank"
70 Added: | T.HEADING (n, s) -> Printf.sprintf "Heading(%d, %S)" n s
71 Added: | T.KEYWORD (k, v) -> Printf.sprintf "Keyword(%S, %S)" k v
72 Added: | T.AFFILIATED_KEYWORD (n, _, v) -> Printf.sprintf "Affiliated(%S, %S)" n v
73 Added: | T.BEGIN_BLOCK (n, p) ->
73 74 Printf.sprintf "Begin_block(%S, %s)" n
74 75 (match p with Some s -> Printf.sprintf "%S" s | None -> "None")
75 Removed: | L.End_block n -> Printf.sprintf "End_block(%S)" n
76 Removed: | L.Drawer_begin n -> Printf.sprintf "Drawer_begin(%S)" n
77 Removed: | L.Drawer_end -> "Drawer_end"
78 Removed: | L.Property (n, v) ->
76 Added: | T.END_BLOCK n -> Printf.sprintf "End_block(%S)" n
77 Added: | T.DRAWER_BEGIN n -> Printf.sprintf "Drawer_begin(%S)" n
78 Added: | T.DRAWER_END -> "Drawer_end"
79 Added: | T.PROPERTY (n, v) ->
79 80 Printf.sprintf "Property(%S, %s)" n
80 81 (match v with Some s -> Printf.sprintf "%S" s | None -> "None")
81 Removed: | L.Planning _ -> "Planning"
82 Removed: | L.List_item d ->
82 Added: | T.PLANNING _ -> "Planning"
83 Added: | T.LIST_ITEM d ->
83 84 Printf.sprintf "List_item(indent=%d, bullet=%S)" d.lid_indent d.lid_bullet
84 Removed: | L.Table_row _ -> "Table_row"
85 Removed: | L.Fixed_width s -> Printf.sprintf "Fixed_width(%S)" s
86 Removed: | L.Comment_line s -> Printf.sprintf "Comment_line(%S)" s
87 Removed: | L.Horizontal_rule -> "Horizontal_rule"
88 Removed: | L.Text_line s -> Printf.sprintf "Text_line(%S)" s
89 Removed: | L.Eof -> "Eof"
85 Added: | T.TABLE_ROW _ -> "Table_row"
86 Added: | T.FIXED_WIDTH s -> Printf.sprintf "Fixed_width(%S)" s
87 Added: | T.COMMENT_LINE s -> Printf.sprintf "Comment_line(%S)" s
88 Added: | T.HORIZONTAL_RULE -> "Horizontal_rule"
89 Added: | T.TEXT_LINE s -> Printf.sprintf "Text_line(%S)" s
90 Added: | T.EOF -> "Eof"
90 91
91 92 let check_tok msg expected actual =
92 93 Alcotest.(check string) msg expected (tok_to_string actual)