[OCaml] Mobile-friendly clone of cgit.
perf Replace LCS matrix with Myers diff
The quadratic LCS matrix cost O(N*M) time and memory regardless of similarity, so two 1500-line files hit the cap even for a one-line change. Myers' O(ND) algorithm — git's default — scales with the number of differences instead. The common prefix and suffix are trimmed first and lines are interned, so a small change in a large file now yields an exact diff. Pairs differing in more than 1024 lines degrade to wholesale replacement, bounding the backtracking trace at about 8 MB. Tests: existing diff expectations unchanged; new tests cover a two-edit diff in a 3000-line file and reconstruction of both inputs from the script, including past the cap.
Changed files
lib/line_diff.ml
@@ -1,11 +1,13 @@
1
1
(** Line-oriented diffs between two blobs.
2
2
3
Removed:
Computes a longest-common-subsequence diff and groups the result into hunks
4
Removed:
with the surrounding context, which is the shape a reader expects from
5
Removed:
[git diff]. Large inputs fall back to a coarser result rather than spending
6
Removed:
unbounded time and memory on the LCS matrix: when the matrix would exceed
7
Removed:
[max_matrix_cells] cells, the diff lists every old line as deleted
8
Removed:
followed by every new line as added.
3
Added:
Computes a Myers diff — the greedy O(ND) algorithm from "An O(ND)
4
Added:
Difference Algorithm and Its Variations" (Myers, 1986), the same algorithm
5
Added:
[git diff] uses by default — and groups the result into hunks with the
6
Added:
surrounding context. Cost scales with the number of differences, not with
7
Added:
file size, so a small change in a large file is cheap. When two files
8
Added:
differ in more than [max_differences] lines after trimming their common
9
Added:
prefix and suffix, the diff falls back to listing every remaining old line
10
Added:
as deleted followed by every remaining new line as added.
9
11
10
12
The module is named for the granularity it works at: whole lines, matched as
11
13
opaque units, with no word- or character-level refinement.
@@ -48,92 +50,214 @@
48
50
List.rev lines |> List.tl |> List.rev
49
51
else lines
50
52
51
Removed:
(* One matrix cell is one word, so the cap bounds a single diff's matrix at
52
Removed:
about 8 MB on a 64-bit system. Concurrent requests each allocate their own
53
Removed:
matrix, which is why the bound stays modest. Beyond it the diff degrades to
54
Removed:
all-deletions-then-all-additions rather than failing. *)
55
Removed:
let max_matrix_cells = 1_000_000
53
Added:
(* The cap on the edit distance D the search explores. The backtracking trace
54
Added:
grows as D² words — about 8 MB at this cap — and the forward pass costs
55
Added:
O((N+M)·D), so the cap bounds both for pathological pairs. A commit that
56
Added:
changes over a thousand lines in one file (after prefix/suffix trimming)
57
Added:
degrades to all-deletions-then-all-additions rather than failing. *)
58
Added:
let max_differences = 1_024
56
59
60
Added:
(* One edit-script step over the trimmed middle of the two files. *)
61
Added:
type edit = Keep | Delete | Insert
62
Added:
63
Added:
(* Myers' greedy shortest-edit-script search over two int arrays (lines are
64
Added:
interned before the search, so equality is an integer compare).
65
Added:
66
Added:
Variables follow the paper so the code can be checked against it: the edit
67
Added:
graph runs from (0, 0) to (n, m), [d] is the edit distance explored so far,
68
Added:
[k = x - y] indexes diagonals, and [v] holds, per diagonal, the furthest x
69
Added:
reached. Each round snapshots the slice of [v] its backtrack step needs
70
Added:
(diagonals -(d+1)..d+1), so trace memory is O(D²) rather than O(D·(N+M)).
71
Added:
72
Added:
Returns the edit script in order, or [None] when the files differ in more
73
Added:
than [max_differences] lines. *)
74
Added:
let myers old_ids new_ids =
75
Added:
let n = Array.length old_ids and m = Array.length new_ids in
76
Added:
let bound = min (n + m) max_differences in
77
Added:
let offset = bound + 1 in
78
Added:
let v = Array.make ((2 * bound) + 3) 0 in
79
Added:
let trace = ref [] in
80
Added:
let exception Found of int in
81
Added:
let found_d =
82
Added:
try
83
Added:
for d = 0 to bound do
84
Added:
let slice =
85
Added:
Array.init ((2 * d) + 3) (fun i -> v.(i - (d + 1) + offset))
86
Added:
in
87
Added:
trace := slice :: !trace;
88
Added:
let k = ref (-d) in
89
Added:
while !k <= d do
90
Added:
let start_x =
91
Added:
if
92
Added:
!k = -d
93
Added:
|| (!k <> d && v.(!k - 1 + offset) < v.(!k + 1 + offset))
94
Added:
then v.(!k + 1 + offset)
95
Added:
else v.(!k - 1 + offset) + 1
96
Added:
in
97
Added:
let x = ref start_x in
98
Added:
let y = ref (start_x - !k) in
99
Added:
while !x < n && !y < m && old_ids.(!x) = new_ids.(!y) do
100
Added:
incr x;
101
Added:
incr y
102
Added:
done;
103
Added:
v.(!k + offset) <- !x;
104
Added:
if !x >= n && !y >= m then raise (Found d);
105
Added:
k := !k + 2
106
Added:
done
107
Added:
done;
108
Added:
None
109
Added:
with Found d -> Some d
110
Added:
in
111
Added:
match found_d with
112
Added:
| None -> None
113
Added:
| Some found_d ->
114
Added:
(* Walk back from (n, m), reading each round's snapshot to find where
115
Added:
the path entered it: a vertical step is an insertion, a horizontal
116
Added:
step a deletion, and the diagonal run before it is kept lines. *)
117
Added:
let edits = ref [] in
118
Added:
let x = ref n and y = ref m in
119
Added:
let rec go d = function
120
Added:
| [] -> ()
121
Added:
| slice :: earlier ->
122
Added:
let slice_offset = d + 1 in
123
Added:
let k = !x - !y in
124
Added:
let previous_k =
125
Added:
if
126
Added:
k = -d
127
Added:
|| (k <> d
128
Added:
&& slice.(k - 1 + slice_offset)
129
Added:
< slice.(k + 1 + slice_offset))
130
Added:
then k + 1
131
Added:
else k - 1
132
Added:
in
133
Added:
let previous_x = slice.(previous_k + slice_offset) in
134
Added:
let previous_y = previous_x - previous_k in
135
Added:
while !x > previous_x && !y > previous_y do
136
Added:
edits := Keep :: !edits;
137
Added:
decr x;
138
Added:
decr y
139
Added:
done;
140
Added:
if d > 0 then
141
Added:
edits := (if !x = previous_x then Insert else Delete) :: !edits;
142
Added:
x := previous_x;
143
Added:
y := previous_y;
144
Added:
go (d - 1) earlier
145
Added:
in
146
Added:
go found_d !trace;
147
Added:
Some !edits
148
Added:
57
149
let of_contents old_content new_content =
58
150
let old_lines = Array.of_list (split_lines old_content) in
59
151
let new_lines = Array.of_list (split_lines new_content) in
60
152
let old_length = Array.length old_lines in
61
153
let new_length = Array.length new_lines in
62
Removed:
let matrix_size = old_length * new_length in
63
Removed:
let rec all_deletions index acc =
64
Removed:
if index = old_length then List.rev acc
65
Removed:
else
66
Removed:
all_deletions (index + 1)
67
Removed:
({
68
Removed:
kind = Deletion;
69
Removed:
old_number = Some (index + 1);
70
Removed:
new_number = None;
71
Removed:
text = old_lines.(index);
72
Removed:
}
73
Removed:
:: acc)
154
Added:
(* Trim the common prefix and suffix before searching: they are context by
155
Added:
definition, and the difference cap should apply to what actually
156
Added:
differs. *)
157
Added:
let prefix = ref 0 in
158
Added:
while
159
Added:
!prefix < old_length
160
Added:
&& !prefix < new_length
161
Added:
&& old_lines.(!prefix) = new_lines.(!prefix)
162
Added:
do
163
Added:
incr prefix
164
Added:
done;
165
Added:
let prefix = !prefix in
166
Added:
let suffix = ref 0 in
167
Added:
while
168
Added:
!suffix < old_length - prefix
169
Added:
&& !suffix < new_length - prefix
170
Added:
&& old_lines.(old_length - 1 - !suffix) = new_lines.(new_length - 1 - !suffix)
171
Added:
do
172
Added:
incr suffix
173
Added:
done;
174
Added:
let suffix = !suffix in
175
Added:
let middle_old = old_length - prefix - suffix in
176
Added:
let middle_new = new_length - prefix - suffix in
177
Added:
(* Intern the middle lines so the search compares integers, not strings. *)
178
Added:
let ids = Hashtbl.create 64 in
179
Added:
let next_id = ref 0 in
180
Added:
let id_of line =
181
Added:
match Hashtbl.find_opt ids line with
182
Added:
| Some id -> id
183
Added:
| None ->
184
Added:
let id = !next_id in
185
Added:
incr next_id;
186
Added:
Hashtbl.add ids line id;
187
Added:
id
74
188
in
75
Removed:
let rec all_additions index acc =
76
Removed:
if index = new_length then List.rev acc
77
Removed:
else
78
Removed:
all_additions (index + 1)
79
Removed:
({
80
Removed:
kind = Addition;
81
Removed:
old_number = None;
82
Removed:
new_number = Some (index + 1);
83
Removed:
text = new_lines.(index);
84
Removed:
}
85
Removed:
:: acc)
189
Added:
let old_ids = Array.init middle_old (fun i -> id_of old_lines.(prefix + i)) in
190
Added:
let new_ids = Array.init middle_new (fun i -> id_of new_lines.(prefix + i)) in
191
Added:
let edits =
192
Added:
match myers old_ids new_ids with
193
Added:
| Some edits -> edits
194
Added:
| None ->
195
Added:
(* Too many differences: list the middle as wholesale replacement. *)
196
Added:
List.init middle_old (fun _ -> Delete)
197
Added:
@ List.init middle_new (fun _ -> Insert)
86
198
in
87
Removed:
if matrix_size > max_matrix_cells then all_deletions 0 [] @ all_additions 0 []
88
Removed:
else
89
Removed:
let lengths = Array.make_matrix (old_length + 1) (new_length + 1) 0 in
90
Removed:
for old_index = old_length - 1 downto 0 do
91
Removed:
for new_index = new_length - 1 downto 0 do
92
Removed:
lengths.(old_index).(new_index) <-
93
Removed:
(if old_lines.(old_index) = new_lines.(new_index) then
94
Removed:
lengths.(old_index + 1).(new_index + 1) + 1
95
Removed:
else
96
Removed:
max
97
Removed:
lengths.(old_index + 1).(new_index)
98
Removed:
lengths.(old_index).(new_index + 1))
99
Removed:
done
100
Removed:
done;
101
Removed:
let rec build old_index new_index acc =
102
Removed:
if old_index = old_length then List.rev acc @ all_additions new_index []
103
Removed:
else if new_index = new_length then
104
Removed:
List.rev acc @ all_deletions old_index []
105
Removed:
else if old_lines.(old_index) = new_lines.(new_index) then
106
Removed:
build (old_index + 1) (new_index + 1)
107
Removed:
({
108
Removed:
kind = Context;
109
Removed:
old_number = Some (old_index + 1);
110
Removed:
new_number = Some (new_index + 1);
111
Removed:
text = old_lines.(old_index);
112
Removed:
}
113
Removed:
:: acc)
114
Removed:
else if
115
Removed:
lengths.(old_index + 1).(new_index)
116
Removed:
>= lengths.(old_index).(new_index + 1)
117
Removed:
then
118
Removed:
build (old_index + 1) new_index
119
Removed:
({
120
Removed:
kind = Deletion;
121
Removed:
old_number = Some (old_index + 1);
122
Removed:
new_number = None;
123
Removed:
text = old_lines.(old_index);
124
Removed:
}
125
Removed:
:: acc)
126
Removed:
else
127
Removed:
build old_index (new_index + 1)
128
Removed:
({
129
Removed:
kind = Addition;
130
Removed:
old_number = None;
131
Removed:
new_number = Some (new_index + 1);
132
Removed:
text = new_lines.(new_index);
133
Removed:
}
134
Removed:
:: acc)
135
Removed:
in
136
Removed:
build 0 0 []
199
Added:
(* Reassemble: prefix context, the edit script over the middle, suffix
200
Added:
context, numbering lines in each file as we go. *)
201
Added:
let context index =
202
Added:
{
203
Added:
kind = Context;
204
Added:
old_number = Some (index + 1);
205
Added:
new_number = Some (index + 1);
206
Added:
text = old_lines.(index);
207
Added:
}
208
Added:
in
209
Added:
let prefix_lines = List.init prefix context in
210
Added:
let old_index = ref prefix and new_index = ref prefix in
211
Added:
let middle_lines =
212
Added:
List.map
213
Added:
(fun edit ->
214
Added:
match edit with
215
Added:
| Keep ->
216
Added:
let line =
217
Added:
{
218
Added:
kind = Context;
219
Added:
old_number = Some (!old_index + 1);
220
Added:
new_number = Some (!new_index + 1);
221
Added:
text = old_lines.(!old_index);
222
Added:
}
223
Added:
in
224
Added:
incr old_index;
225
Added:
incr new_index;
226
Added:
line
227
Added:
| Delete ->
228
Added:
let line =
229
Added:
{
230
Added:
kind = Deletion;
231
Added:
old_number = Some (!old_index + 1);
232
Added:
new_number = None;
233
Added:
text = old_lines.(!old_index);
234
Added:
}
235
Added:
in
236
Added:
incr old_index;
237
Added:
line
238
Added:
| Insert ->
239
Added:
let line =
240
Added:
{
241
Added:
kind = Addition;
242
Added:
old_number = None;
243
Added:
new_number = Some (!new_index + 1);
244
Added:
text = new_lines.(!new_index);
245
Added:
}
246
Added:
in
247
Added:
incr new_index;
248
Added:
line)
249
Added:
edits
250
Added:
in
251
Added:
let suffix_lines =
252
Added:
List.init suffix (fun i ->
253
Added:
{
254
Added:
kind = Context;
255
Added:
old_number = Some (old_length - suffix + i + 1);
256
Added:
new_number = Some (new_length - suffix + i + 1);
257
Added:
text = old_lines.(old_length - suffix + i);
258
Added:
})
259
Added:
in
260
Added:
prefix_lines @ middle_lines @ suffix_lines
137
261
138
262
let hunks ?(context = 3) lines =
139
263
let lines = Array.of_list lines in
lib/line_diff.mli
@@ -1,12 +1,13 @@
1
1
(** Line-oriented diffs between two blobs.
2
2
3
Removed:
Computes a longest-common-subsequence diff and groups the result into hunks
4
Removed:
with the surrounding context, which is the shape a reader expects from
5
Removed:
[git diff]. Large inputs fall back to a coarser result rather than spending
6
Removed:
unbounded time and memory on the LCS matrix.
3
Added:
Computes a Myers diff — the algorithm [git diff] uses by default — and
4
Added:
groups the result into hunks with the surrounding context. Cost scales
5
Added:
with the number of differences rather than with file size, so a small
6
Added:
change in a large file is cheap. Two files that differ in more than a
7
Added:
thousand lines fall back to a coarse wholesale-replacement diff.
7
8
8
Removed:
The module is named for the granularity it works at: whole lines, matched as
9
Removed:
opaque units, with no word- or character-level refinement.
9
Added:
The module is named for the granularity it works at: whole lines, matched
10
Added:
as opaque units, with no word- or character-level refinement.
10
11
11
12
It produces data only. Rendering it is {!module:Ui.Diff}'s job. *)
12
13
@@ -53,9 +54,10 @@
53
54
54
55
val of_contents : string -> string -> line list
55
56
(** [of_contents old_content new_content] produces a flat list of diff lines
56
Removed:
between two file contents. Falls back to listing all deletions followed by
57
Removed:
all additions when the LCS matrix would exceed one million cells (about
58
Removed:
8 MB), so one request cannot hold an arbitrarily large matrix. *)
57
Added:
between two file contents, computed with Myers' O(ND) algorithm. The
58
Added:
common prefix and suffix are matched directly; when the remainder differs
59
Added:
in more than 1024 lines, it is listed as all deletions followed by all
60
Added:
additions rather than spending unbounded time on the search. *)
59
61
60
62
val hunks : ?context:int -> line list -> hunk list
61
63
(** Group diff lines into hunks with [context] lines of surrounding unchanged
test/test_diff.ml
@@ -98,10 +98,11 @@
98
98
99
99
let test_large_file_fallback () =
100
100
let open Ogit.Resolvers.Diff in
101
Removed:
(* 2001 * 2001 = 4_004_001 > 4_000_000 threshold *)
101
Added:
(* Every line differs after trimming: 2000 edits exceed the 1024-difference
102
Added:
cap, so the diff degrades to wholesale replacement. *)
102
103
let old_content = String.concat "\n" (List.init 2001 string_of_int) in
103
104
let new_content =
104
Removed:
String.concat "\n" (List.init 2001 (fun i -> string_of_int (i + 1000)))
105
Added:
String.concat "\n" (List.init 2001 (fun i -> string_of_int (i + 3000)))
105
106
in
106
107
let lines = of_contents old_content new_content in
107
108
let has_deletions = List.exists (fun l -> l.kind = Deletion) lines in
@@ -112,6 +113,78 @@
112
113
let has_context = List.exists (fun l -> l.kind = Context) lines in
113
114
Alcotest.(check bool) "no context in fallback" false has_context
114
115
116
Added:
(* Cost tracks the number of differences, not file size: two edits in a
117
Added:
3000-line file must produce an exact diff, which the old
118
Added:
quadratic-matrix implementation could not afford at this size. *)
119
Added:
let test_small_change_in_large_file () =
120
Added:
let open Ogit.Resolvers.Diff in
121
Added:
let numbered = List.init 3000 (fun i -> "line " ^ string_of_int (i + 1)) in
122
Added:
let changed =
123
Added:
List.mapi
124
Added:
(fun i line -> if i = 100 || i = 2900 then line ^ " changed" else line)
125
Added:
numbered
126
Added:
in
127
Added:
let lines =
128
Added:
of_contents
129
Added:
(String.concat "\n" numbered ^ "\n")
130
Added:
(String.concat "\n" changed ^ "\n")
131
Added:
in
132
Added:
let count kind =
133
Added:
List.length (List.filter (fun l -> l.kind = kind) lines)
134
Added:
in
135
Added:
Alcotest.(check int) "two deletions" 2 (count Deletion);
136
Added:
Alcotest.(check int) "two additions" 2 (count Addition);
137
Added:
Alcotest.(check int) "rest is context" 2998 (count Context);
138
Added:
(match List.rev lines with
139
Added:
| last :: _ ->
140
Added:
Alcotest.(check (option int)) "last old number" (Some 3000)
141
Added:
last.old_number;
142
Added:
Alcotest.(check (option int)) "last new number" (Some 3000)
143
Added:
last.new_number
144
Added:
| [] -> Alcotest.fail "no lines")
145
Added:
146
Added:
(* Whatever script the search picks, replaying it must reproduce both inputs:
147
Added:
the old file is the context and deleted lines in order, the new file the
148
Added:
context and added lines. *)
149
Added:
let test_reconstruction () =
150
Added:
let open Ogit.Resolvers.Diff in
151
Added:
let reconstruct_old lines =
152
Added:
List.filter_map
153
Added:
(fun l ->
154
Added:
match l.kind with Context | Deletion -> Some l.text | Addition -> None)
155
Added:
lines
156
Added:
in
157
Added:
let reconstruct_new lines =
158
Added:
List.filter_map
159
Added:
(fun l ->
160
Added:
match l.kind with Context | Addition -> Some l.text | Deletion -> None)
161
Added:
lines
162
Added:
in
163
Added:
let check name old_lines new_lines =
164
Added:
let content lines =
165
Added:
match lines with [] -> "" | _ -> String.concat "\n" lines ^ "\n"
166
Added:
in
167
Added:
let diff = of_contents (content old_lines) (content new_lines) in
168
Added:
Alcotest.(check (list string))
169
Added:
(name ^ ": old reconstructed")
170
Added:
old_lines (reconstruct_old diff);
171
Added:
Alcotest.(check (list string))
172
Added:
(name ^ ": new reconstructed")
173
Added:
new_lines (reconstruct_new diff)
174
Added:
in
175
Added:
check "replace middle" [ "a"; "b"; "c" ] [ "a"; "x"; "c" ];
176
Added:
check "empty old" [] [ "a"; "b" ];
177
Added:
check "empty new" [ "a"; "b" ] [];
178
Added:
check "moved block" [ "a"; "b"; "c"; "d" ] [ "c"; "d"; "a"; "b" ];
179
Added:
check "interleaved"
180
Added:
[ "1"; "2"; "3"; "4"; "5"; "6" ]
181
Added:
[ "1"; "x"; "3"; "y"; "5"; "z" ];
182
Added:
check "repeated lines" [ "a"; "a"; "b"; "a" ] [ "a"; "b"; "a"; "a" ];
183
Added:
(* Past the difference cap the coarse fallback must still reconstruct. *)
184
Added:
check "beyond the cap"
185
Added:
(List.init 600 (fun i -> "old " ^ string_of_int i))
186
Added:
(List.init 700 (fun i -> "new " ^ string_of_int i))
187
Added:
115
188
let suite =
116
189
( "diff",
117
190
[
@@ -128,4 +201,7 @@
128
201
Alcotest.test_case "single hunk" `Quick test_hunks;
129
202
Alcotest.test_case "multiple hunks" `Quick test_multiple_hunks;
130
203
Alcotest.test_case "large file fallback" `Quick test_large_file_fallback;
204
Added:
Alcotest.test_case "small change in large file" `Quick
205
Added:
test_small_change_in_large_file;
206
Added:
Alcotest.test_case "reconstruction" `Quick test_reconstruction;
131
207
] )