Skip to content

Commit 97d84c1

Browse files
gaoconggitgaoconggit
authored
Merge pull request #45 from gaoconggit/feature/fix-code-highlighting-bug
Implement enhanced C# language tokenizer to fix string highlighting i…
2 parents 3cc4605 + 8f7406c commit 97d84c1

4 files changed

Lines changed: 265 additions & 10 deletions

File tree

‎SharpPad/wwwroot/csharpLanguageDefinition.js‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
// Custom C# Language Definition - Fixes string highlighting issues with URLs
2+
// Specifically addresses the problem where https:// inside strings affects .Dump() highlighting
3+
4+
export function overrideCSharpLanguage() {
5+
// Enhanced C# language definition that fixes URL highlighting in strings
6+
const enhancedCSharpLanguage = {
7+
defaultToken: '',
8+
tokenPostfix: '.cs',
9+
10+
keywords: [
11+
'extern', 'alias', 'using', 'bool', 'decimal', 'sbyte', 'byte', 'short',
12+
'ushort', 'int', 'uint', 'long', 'ulong', 'char', 'float', 'double',
13+
'object', 'dynamic', 'string', 'assembly', 'is', 'as', 'ref', 'out',
14+
'this', 'base', 'new', 'typeof', 'void', 'checked', 'unchecked',
15+
'default', 'delegate', 'var', 'const', 'if', 'else', 'switch', 'case',
16+
'while', 'do', 'for', 'foreach', 'in', 'break', 'continue', 'goto',
17+
'return', 'throw', 'try', 'catch', 'finally', 'lock', 'yield', 'from',
18+
'let', 'where', 'join', 'on', 'equals', 'into', 'orderby', 'ascending',
19+
'descending', 'select', 'group', 'by', 'namespace', 'partial', 'class',
20+
'field', 'event', 'method', 'param', 'public', 'protected', 'internal',
21+
'private', 'abstract', 'sealed', 'static', 'struct', 'readonly',
22+
'volatile', 'virtual', 'override', 'params', 'get', 'set', 'add',
23+
'remove', 'operator', 'true', 'false', 'implicit', 'explicit',
24+
'interface', 'enum', 'null', 'async', 'await', 'fixed', 'sizeof',
25+
'stackalloc', 'unsafe', 'nameof', 'when'
26+
],
27+
28+
namespaceFollows: ['namespace', 'using'],
29+
parenFollows: ['if', 'for', 'while', 'switch', 'foreach', 'using', 'catch', 'when'],
30+
operators: [
31+
'=', '??', '||', '&&', '|', '^', '&', '==', '!=', '<=', '>=', '<<',
32+
'+', '-', '*', '/', '%', '!', '~', '++', '--', '+=', '-=', '*=',
33+
'/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '>>', '=>'
34+
],
35+
36+
symbols: /[=><!~?:&|+\-*\/\^%]+/,
37+
escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
38+
39+
tokenizer: {
40+
root: [
41+
// String literals MUST be processed FIRST to prevent // inside strings from being treated as comments
42+
[/"/, { token: 'string.quote', next: '@string' }],
43+
[/\$\@"/, { token: 'string.quote', next: '@litinterpstring' }],
44+
[/\@"/, { token: 'string.quote', next: '@litstring' }],
45+
[/\$"/, { token: 'string.quote', next: '@interpolatedstring' }],
46+
47+
// Character literals
48+
[/'[^\\']'/, 'string'],
49+
[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
50+
[/'/, 'string.invalid'],
51+
52+
// Identifiers and keywords (including method names)
53+
[/\@?[a-zA-Z_]\w*/, {
54+
cases: {
55+
'@namespaceFollows': { token: 'keyword.$0', next: '@namespace' },
56+
'@keywords': { token: 'keyword.$0', next: '@qualified' },
57+
'@default': { token: 'identifier', next: '@qualified' }
58+
}
59+
}],
60+
61+
// Whitespace (includes comments - but only when NOT inside strings)
62+
{ include: '@whitespace' },
63+
64+
// Brackets
65+
[/}/, {
66+
cases: {
67+
'$S2==interpolatedstring': { token: 'string.quote', next: '@pop' },
68+
'$S2==litinterpstring': { token: 'string.quote', next: '@pop' },
69+
'@default': '@brackets'
70+
}
71+
}],
72+
[/[{}()\[\]]/, '@brackets'],
73+
[/[<>](?!@symbols)/, '@brackets'],
74+
75+
// Operators
76+
[/@symbols/, {
77+
cases: {
78+
'@operators': 'delimiter',
79+
'@default': ''
80+
}
81+
}],
82+
83+
// Numbers
84+
[/[0-9_]*\.[0-9_]+([eE][\-+]?\d+)?[fFdD]?/, 'number.float'],
85+
[/0[xX][0-9a-fA-F_]+/, 'number.hex'],
86+
[/0[bB][01_]+/, 'number.hex'],
87+
[/[0-9_]+/, 'number'],
88+
89+
// Delimiters
90+
[/[;,.]/, 'delimiter']
91+
],
92+
93+
qualified: [
94+
[/[a-zA-Z_][\w]*/, {
95+
cases: {
96+
'@keywords': { token: 'keyword.$0' },
97+
'@default': 'identifier'
98+
}
99+
}],
100+
[/\./, 'delimiter'],
101+
['', '', '@pop']
102+
],
103+
104+
namespace: [
105+
{ include: '@whitespace' },
106+
[/[A-Z]\w*/, 'namespace'],
107+
[/[\.=]/, 'delimiter'],
108+
['', '', '@pop']
109+
],
110+
111+
comment: [
112+
[/[^\/*]+/, 'comment'],
113+
['\\*/', 'comment', '@pop'],
114+
[/[\/*]/, 'comment']
115+
],
116+
117+
// String state - NO comment processing allowed here
118+
string: [
119+
[/[^\\"]+/, 'string'], // Everything inside string is just string content
120+
[/@escapes/, 'string.escape'],
121+
[/\\./, 'string.escape.invalid'],
122+
[/"/, { token: 'string.quote', next: '@pop' }]
123+
],
124+
125+
litstring: [
126+
[/[^"]+/, 'string'], // Everything inside literal string is just string content
127+
[/""/, 'string.escape'],
128+
[/"/, { token: 'string.quote', next: '@pop' }]
129+
],
130+
131+
litinterpstring: [
132+
[/[^"{]+/, 'string'],
133+
[/""/, 'string.escape'],
134+
[/{{/, 'string.escape'],
135+
[/}}/, 'string.escape'],
136+
[/{/, { token: 'string.quote', next: 'root.litinterpstring' }],
137+
[/"/, { token: 'string.quote', next: '@pop' }]
138+
],
139+
140+
interpolatedstring: [
141+
[/[^\\"{]+/, 'string'],
142+
[/@escapes/, 'string.escape'],
143+
[/\\./, 'string.escape.invalid'],
144+
[/{{/, 'string.escape'],
145+
[/}}/, 'string.escape'],
146+
[/{/, { token: 'string.quote', next: 'root.interpolatedstring' }],
147+
[/"/, { token: 'string.quote', next: '@pop' }]
148+
],
149+
150+
// Whitespace handling - comments only processed when not in string context
151+
whitespace: [
152+
[/^[ \t\v\f]*#((r)|(load))(?=\s)/, 'directive.csx'],
153+
[/^[ \t\v\f]*#\w.*$/, 'namespace.cpp'],
154+
[/[ \t\v\f\r\n]+/, ''],
155+
[/\/\*/, 'comment', '@comment'],
156+
[/\/\/.*$/, 'comment']
157+
]
158+
}
159+
};
160+
161+
// Override the existing C# language tokenizer
162+
const languageId = 'csharp';
163+
let overrideApplied = false;
164+
165+
const applyOverride = () => {
166+
if (!globalThis.monaco?.languages?.setMonarchTokensProvider) {
167+
return false;
168+
}
169+
170+
try {
171+
globalThis.monaco.languages.setMonarchTokensProvider(languageId, enhancedCSharpLanguage);
172+
if (!overrideApplied) {
173+
overrideApplied = true;
174+
console.log('C# language tokenizer successfully overridden - fixed string URL highlighting');
175+
}
176+
return true;
177+
} catch (error) {
178+
console.error('Failed to override C# language tokenizer:', error);
179+
return false;
180+
}
181+
};
182+
183+
const scheduleOverride = (delay = 0, attemptsLeft = 10) => {
184+
if (attemptsLeft <= 0) {
185+
return;
186+
}
187+
188+
setTimeout(() => {
189+
const applied = applyOverride();
190+
if (!applied) {
191+
scheduleOverride(Math.min(delay + 50, 250), attemptsLeft - 1);
192+
}
193+
}, delay);
194+
};
195+
196+
try {
197+
if (globalThis.monaco?.languages?.getLanguages?.().some(lang => lang.id === languageId)) {
198+
scheduleOverride();
199+
} else {
200+
scheduleOverride(25);
201+
}
202+
} catch (error) {
203+
console.warn('Unable to verify Monaco languages for override immediately:', error);
204+
scheduleOverride();
205+
}
206+
207+
if (globalThis.monaco?.languages?.onLanguage) {
208+
globalThis.monaco.languages.onLanguage(languageId, () => {
209+
scheduleOverride();
210+
scheduleOverride(50);
211+
scheduleOverride(200);
212+
});
213+
} else {
214+
// Fallback: keep trying until Monaco exposes the language APIs
215+
scheduleOverride();
216+
}
217+
}
218+
219+

‎SharpPad/wwwroot/csharpLanguageProvider.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import { getCurrentFile, shouldUseMultiFileMode, createMultiFileRequest, createSingleFileRequest } from './utils/common.js';
22
import { sendRequest } from './utils/apiService.js';
3+
import { overrideCSharpLanguage } from './csharpLanguageDefinition.js';
34

45
export function registerCsharpProvider() {
5-
// Monaco Editor 已内置 C# 语言支持
6-
// 完全使用原生语法高亮，不添加任何增强着色
6+
// Override Monaco's built-in C# language with enhanced tokenizer
7+
// This specifically fixes the issue where https:// inside strings affects .Dump() highlighting
8+
overrideCSharpLanguage();
79

810
monaco.languages.registerCompletionItemProvider('csharp', {
911
triggerCharacters: [".", " "],

‎SharpPad/wwwroot/semanticColoring.js‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -289,18 +289,23 @@ function setupModelSemanticColoring(model, legend) {
289289
*/
290290
async function applySemanticDecorationsFast(model, tokenData, legend, decorationIds) {
291291
const lines = model.getLinesContent();
292+
const stringRanges = detectStringRanges(model);
292293
const commentLines = new Set();
293-
294-
// 极速注释行扫描
294+
295+
// 粗略注释扫描（忽略字符串内的 //）
295296
for (let i = 0; i < lines.length; i++) {
296-
if (lines[i].includes('//')) {
297-
commentLines.add(i + 1);
297+
const line = lines[i];
298+
const commentIndex = line.indexOf("//");
299+
if (commentIndex !== -1) {
300+
const commentPos = { line: i + 1, char: commentIndex };
301+
if (!isPositionInString(commentPos, stringRanges)) {
302+
commentLines.add(i + 1);
303+
}
298304
}
299305
}
300306

301-
// 如果有 Worker 且数据量大，使用 Worker 处理
307+
// 如果有 Worker 支持且数据量较大，使用 Worker 处理
302308
if (semanticWorker && tokenData.length > 1000) {
303-
const stringRanges = detectStringRanges(model);
304309

305310
return new Promise((resolve) => {
306311
const handleWorkerMessage = (e) => {
@@ -330,7 +335,6 @@ async function applySemanticDecorationsFast(model, tokenData, legend, decoration
330335
let currentChar = 0;
331336

332337
// 检测字符串范围以避免在字符串内部应用语义着色
333-
const stringRanges = detectStringRanges(model);
334338

335339
// 只处理关键令牌类型，添加更多重要的类型
336340
const importantTypes = new Set(['class', 'interface', 'method', 'function', 'comment', 'type', 'struct', 'enum']);
@@ -344,7 +348,13 @@ async function applySemanticDecorationsFast(model, tokenData, legend, decoration
344348

345349
// 更精确的注释检测
346350
const lineContent = model.getLineContent(currentLine + 1);
347-
const commentStart = lineContent.indexOf('//');
351+
let commentStart = lineContent.indexOf("//");
352+
if (commentStart !== -1) {
353+
const commentPos = { line: currentLine + 1, char: commentStart };
354+
if (isPositionInString(commentPos, stringRanges)) {
355+
commentStart = -1;
356+
}
357+
}
348358
const isInComment = commentStart !== -1 && currentChar >= commentStart;
349359

350360
// 检查是否在字符串内部

‎test-syntax-highlighting.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
using System;
2+
3+
class Program
4+
{
5+
public static void Main()
6+
{
7+
// 这行代码应该展示正确的语法着色
8+
"Hello, SharpPad! 关注我: https://github.com/gaoconggit/SharpPad".Dump();
9+
10+
// 另一个测试案例
11+
string message = "Visit: https://github.com/microsoft/monaco-editor";
12+
message.Dump();
13+
14+
// 确保普通注释仍然正常工作
15+
// This is a regular comment and should be highlighted as comment
16+
17+
/*
18+
* Block comment test with URL: https://example.com
19+
* Should all be highlighted as comment
20+
*/
21+
22+
Console.WriteLine("Regular method calls should work normally");
23+
}
24+
}

0 commit comments

Comments
 (0)