Compare commits

...
Author SHA1 Message Date
Wolfsblvt 7ee041c11d fix: handle nested same-delimiter emphasis in markdown rendering
Showdown cannot natively process nested emphasis with the same delimiter
(e.g. *outer *inner* outer* or **outer **inner** outer**), rendering
them as broken text. Add a pre-processing step that detects nesting using
stack-based pairing with CommonMark-like flanking rules:

- Nested *...* inside *...*: promote inner to ** (bold-inside-italic)
- Nested **...** inside **...**:  strip inner ** (redundant bold)

Adds showdown-emphasis.js combining the underscore extension with the new
fixNestedEmphasis pre-processor, called in messageFormatting before
fixMarkdown. Includes 27 Playwright e2e tests.

Closes #5500
2026-05-16 05:20:07 +02:00
3 changed files with 490 additions and 1 deletions
+7 -1
View File
@@ -215,7 +215,7 @@ import {
} from './scripts/tags.js';
import { checkOpenRouterAuth, initSecrets, readSecretState } from './scripts/secrets.js';
import { markdownExclusionExt } from './scripts/showdown-exclusion.js';
import { markdownUnderscoreExt } from './scripts/showdown-underscore.js';
import { markdownUnderscoreExt, fixNestedEmphasis } from './scripts/showdown-emphasis.js';
import { NOTE_MODULE_NAME, initAuthorsNote, metadata_keys, setFloatingPrompt, shouldWIAddPrompt } from './scripts/authors-note.js';
import { registerPromptManagerMigration } from './scripts/PromptManager.js';
import { getRegexedString, regex_placement } from './scripts/extensions/regex/engine.js';
@@ -1813,6 +1813,12 @@ export function messageFormatting(mes, ch_name, isSystem, isUser, messageId, san
});
}
// Convert nested same-delimiter emphasis to bold before any markdown fixing.
// Must run before fixMarkdown which would strip the spaces that indicate nesting.
if (!isSystem) {
mes = fixNestedEmphasis(mes);
}
if (power_user.auto_fix_generated_markdown) {
mes = fixMarkdown(mes, true);
}
+237
View File
@@ -0,0 +1,237 @@
/**
* Showdown emphasis extensions and pre-processing utilities.
*
* Contains:
* - markdownUnderscoreExt: Showdown output extension for _word_ emphasis
* - fixNestedEmphasis: Pre-processor to convert nested same-delimiter emphasis to bold
*/
// ===== Underscore Emphasis Extension =====
/**
* Showdown extension that replaces words surrounded by singular underscores with <em> tags.
* @returns {import('showdown').ShowdownExtension[]} An array of Showdown extensions
*/
export const markdownUnderscoreExt = () => {
try {
if (!canUseNegativeLookbehind()) {
console.log('Showdown-underscore extension: Negative lookbehind not supported. Skipping.');
return [];
}
return [{
type: 'output',
regex: new RegExp('(<code(?:\\s+[^>]*)?>[\\s\\S]*?<\\/code>|<style(?:\\s+[^>]*)?>[\\s\\S]*?<\\/style>)|\\b(?<!_)_(?!_)(.*?)(?<!_)_(?!_)\\b', 'gi'),
replace: function (match, tagContent, italicContent) {
if (tagContent) {
// If it's inside <code> or <style> tags, return unchanged
return match;
} else if (italicContent) {
// If it's an italic group, apply the replacement
return '<em>' + italicContent + '</em>';
}
// If none of the conditions are met, return the original match
return match;
},
}];
} catch (e) {
console.error('Error in Showdown-underscore extension:', e);
return [];
}
};
function canUseNegativeLookbehind() {
try {
new RegExp('(?<!_)');
return true;
} catch (e) {
return false;
}
}
// ===== Nested Emphasis Pre-Processor =====
/**
* Fixes nested emphasis that showdown cannot handle.
*
* Showdown cannot handle nested emphasis with the same delimiter character.
* This function handles two cases:
* 1. `*outer *inner* outer*` — inner `*` promoted to `**` (renders as bold-inside-italic)
* 2. `**outer **inner** outer**` — inner `**` stripped (bold-inside-bold is redundant)
*
* Uses CommonMark-like flanking rules and stack-based pairing to detect nesting.
*
* @param {string} text The message text to process
* @returns {string} The processed text with nested emphasis fixed
* @example
* fixNestedEmphasis("*She sighs. *I can't do this,* she says.*")
* // → "*She sighs. **I can't do this,** she says.*"
* fixNestedEmphasis("**This is **bold**, isn't it?**")
* // → "**This is bold, isn't it?**"
*/
export function fixNestedEmphasis(text) {
// Temporarily replace code blocks and inline code to protect them from modification
const codeBlocks = [];
let processed = text.replace(/```[\s\S]*?```|~~~[\s\S]*?~~~|``[\s\S]*?``|`[^`\n]*`/g, (match) => {
codeBlocks.push(match);
return `\x00CODE_BLOCK_${codeBlocks.length - 1}\x00`;
});
processed = processed.split('\n').map(line => {
// Fix nested ** inside ** first (strip inner), then nested * inside * (promote inner)
line = fixNestedBoldLine(line);
line = fixNestedItalicLine(line);
return line;
}).join('\n');
// Restore code blocks
processed = processed.replace(/\x00CODE_BLOCK_(\d+)\x00/g, (_, idx) => codeBlocks[parseInt(idx)]);
return processed;
}
/**
* Finds nested `**...**` pairs inside other `**...**` pairs and strips the inner delimiters.
* Bold inside bold is redundant — like GitHub, the whole block renders as bold.
* @param {string} line A single line of text
* @returns {string} The processed line
*/
function fixNestedBoldLine(line) {
// Find all ** positions (exactly two consecutive asterisks, not part of ***)
const positions = [];
for (let i = 0; i < line.length - 1; i++) {
if (line[i] === '*' && line[i + 1] === '*') {
const before = i > 0 ? line[i - 1] : '';
const after = i + 2 < line.length ? line[i + 2] : '';
// Must be exactly ** (not *** or more)
if (before !== '*' && after !== '*') {
positions.push(i);
i++; // skip the second *
}
}
}
// Need at least 4 ** delimiters for nesting to be possible
if (positions.length < 4) return line;
// Stack-based pairing with flanking rules
const stack = [];
const pairs = [];
for (let i = 0; i < positions.length; i++) {
const pos = positions[i];
// For **, check the char before the first * and after the second *
const charBefore = pos > 0 ? line[pos - 1] : '';
const charAfter = pos + 2 < line.length ? line[pos + 2] : '';
const canClose = charBefore !== '' && !/\s/.test(charBefore);
const canOpen = charAfter !== '' && !/\s/.test(charAfter);
if (canClose && stack.length > 0) {
const openIdx = stack.pop();
pairs.push([openIdx, i]);
} else if (canOpen) {
stack.push(i);
}
}
// Determine which pairs are nested inside others
const nestedPositions = new Set();
for (let i = 0; i < pairs.length; i++) {
const [outerOpen, outerClose] = [positions[pairs[i][0]], positions[pairs[i][1]]];
for (let j = 0; j < pairs.length; j++) {
if (i === j) continue;
const [innerOpen, innerClose] = [positions[pairs[j][0]], positions[pairs[j][1]]];
if (innerOpen > outerOpen && innerClose < outerClose) {
// Inner pair is nested — mark both its ** delimiters for removal
nestedPositions.add(innerOpen);
nestedPositions.add(innerClose);
}
}
}
if (nestedPositions.size === 0) return line;
// Build result, skipping the nested ** delimiters
let result = '';
for (let i = 0; i < line.length; i++) {
if (nestedPositions.has(i)) {
i++; // skip both characters of **
continue;
}
result += line[i];
}
return result;
}
/**
* Finds nested `*...*` pairs inside other `*...*` pairs and promotes them to `**...**`.
* This converts inner emphasis to bold so showdown renders bold-inside-italic.
* @param {string} line A single line of text
* @returns {string} The processed line
*/
function fixNestedItalicLine(line) {
// Find all single-* positions (not part of ** or ***)
const positions = [];
for (let i = 0; i < line.length; i++) {
if (line[i] === '*') {
const before = i > 0 ? line[i - 1] : '';
const after = i < line.length - 1 ? line[i + 1] : '';
if (before !== '*' && after !== '*') {
positions.push(i);
}
}
}
// Need at least 4 single asterisks for nesting to be possible
if (positions.length < 4) return line;
// Use stack-based pairing with CommonMark-like flanking rules:
// - An asterisk can OPEN emphasis if followed by a non-whitespace character
// - An asterisk can CLOSE emphasis if preceded by a non-whitespace character
const stack = [];
const pairs = [];
for (let i = 0; i < positions.length; i++) {
const pos = positions[i];
const charBefore = pos > 0 ? line[pos - 1] : '';
const charAfter = pos < line.length - 1 ? line[pos + 1] : '';
const canClose = charBefore !== '' && !/\s/.test(charBefore);
const canOpen = charAfter !== '' && !/\s/.test(charAfter);
// Prefer closing over opening (matches CommonMark behavior)
if (canClose && stack.length > 0) {
const openIdx = stack.pop();
pairs.push([openIdx, i]);
} else if (canOpen) {
stack.push(i);
}
}
// Determine nesting: a pair is nested if fully contained within another pair
const nestedIndices = new Set();
for (let i = 0; i < pairs.length; i++) {
const [outerOpen, outerClose] = [positions[pairs[i][0]], positions[pairs[i][1]]];
for (let j = 0; j < pairs.length; j++) {
if (i === j) continue;
const [innerOpen, innerClose] = [positions[pairs[j][0]], positions[pairs[j][1]]];
if (innerOpen > outerOpen && innerClose < outerClose) {
nestedIndices.add(innerOpen);
nestedIndices.add(innerClose);
}
}
}
if (nestedIndices.size === 0) return line;
// Build result with doubled asterisks at nested positions
let result = '';
for (let i = 0; i < line.length; i++) {
result += line[i];
if (nestedIndices.has(i)) {
result += '*';
}
}
return result;
}
+246
View File
@@ -0,0 +1,246 @@
import { test, expect } from '@playwright/test';
import { testSetup } from './frontent-test-utils.js';
test.describe('fixNestedEmphasis', () => {
test.beforeEach(testSetup.goST);
test.describe('Basic nested emphasis', () => {
test('should convert inner emphasis to bold within outer emphasis', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*She looks at you. *Are you okay?* she asks.*');
expect(result).toBe('*She looks at you. **Are you okay?** she asks.*');
});
test('should handle the issue example case', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*Some Text *Nested Emphasis Text* More text*');
expect(result).toBe('*Some Text **Nested Emphasis Text** More text*');
});
test('should handle short nested action/speech', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*action *speech* action*');
expect(result).toBe('*action **speech** action*');
});
test('should handle nested emphasis with punctuation in inner text', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*She sighs. *I never asked for this,* she mutters, turning away.*');
expect(result).toBe('*She sighs. **I never asked for this,** she mutters, turning away.*');
});
});
test.describe('Multiple nested pairs', () => {
test('should handle multiple nested pairs in same outer emphasis', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*action *speech1* middle *speech2* end*');
expect(result).toBe('*action **speech1** middle **speech2** end*');
});
test('should handle multiple nested pairs with longer text', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*She said *hello* and *goodbye* to him.*');
expect(result).toBe('*She said **hello** and **goodbye** to him.*');
});
});
test.describe('Text that should NOT be modified', () => {
test('should not modify simple emphasis', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*normal emphasis*');
expect(result).toBe('*normal emphasis*');
});
test('should not modify bold text', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '**bold text**');
expect(result).toBe('**bold text**');
});
test('should not modify sequential (non-nested) emphasis', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*first emphasis* and *second emphasis*');
expect(result).toBe('*first emphasis* and *second emphasis*');
});
test('should not modify already-correct bold inside italic', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*text **already bold** text*');
expect(result).toBe('*text **already bold** text*');
});
test('should not modify triple emphasis', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '***triple emphasis***');
expect(result).toBe('***triple emphasis***');
});
test('should not modify plain text', async ({ page }) => {
const result = await runFixNestedEmphasis(page, 'no formatting at all');
expect(result).toBe('no formatting at all');
});
test('should not modify adjacent emphasis without spaces (non-nested pattern)', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*hello*world*bye*');
expect(result).toBe('*hello*world*bye*');
});
});
test.describe('Nested bold (** inside **)', () => {
test('should strip inner ** when nested inside outer **', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '**This is **bold**, is it?**');
expect(result).toBe('**This is bold, is it?**');
});
test('should strip inner ** for simple nesting', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '**text **nested** text**');
expect(result).toBe('**text nested text**');
});
test('should strip inner ** for short content', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '**a **b** c**');
expect(result).toBe('**a b c**');
});
test('should strip inner ** with longer word', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '**She said **hello** to him**');
expect(result).toBe('**She said hello to him**');
});
test('should handle multiple nested ** pairs', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '**She **whispered** and then **shouted** at him**');
expect(result).toBe('**She whispered and then shouted at him**');
});
test('should not modify normal bold (no nesting)', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '**normal bold**');
expect(result).toBe('**normal bold**');
});
test('should not modify sequential bold', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '**first** and **second**');
expect(result).toBe('**first** and **second**');
});
});
test.describe('Edge cases', () => {
test('should handle nested emphasis next to sequential emphasis', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*start *nested* end* and *another*');
expect(result).toBe('*start **nested** end* and *another*');
});
test('should not modify asterisk preceded by space (non-flanking)', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*single asterisk at end *');
expect(result).toBe('*single asterisk at end *');
});
test('should handle nested emphasis adjacent to punctuation', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*She said: *"Hello!"* quietly.*');
expect(result).toBe('*She said: **"Hello!"** quietly.*');
});
test('should handle tabs as whitespace around nested emphasis', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*She looks.\t*Are you okay?*\tshe asks.*');
expect(result).toBe('*She looks.\t**Are you okay?**\tshe asks.*');
});
test('should process each line independently for multi-line text', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '*first line\n*nested* second line*');
// Lines are processed independently - first line has only 1 asterisk, second has 3
expect(result).toBe('*first line\n*nested* second line*');
});
});
test.describe('Code block protection', () => {
test('should not modify content inside inline code', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '`*code *nested* code*`');
expect(result).toBe('`*code *nested* code*`');
});
test('should not modify content inside fenced code blocks', async ({ page }) => {
const result = await runFixNestedEmphasis(page, '```\n*code *nested* code*\n```');
expect(result).toBe('```\n*code *nested* code*\n```');
});
});
test.describe('Realistic roleplay examples', () => {
test('should handle dialogue within action narration', async ({ page }) => {
const input = '*She reaches out, her hand trembling. *Please... don\'t leave me,* she whispers, tears streaming down her face.*';
const expected = '*She reaches out, her hand trembling. **Please... don\'t leave me,** she whispers, tears streaming down her face.*';
const result = await runFixNestedEmphasis(page, input);
expect(result).toBe(expected);
});
test('should handle multiple dialogue lines within narration', async ({ page }) => {
const input = '*The door creaks open. *Who\'s there?* a voice calls out from the darkness. *Show yourself!* it demands.*';
const expected = '*The door creaks open. **Who\'s there?** a voice calls out from the darkness. **Show yourself!** it demands.*';
const result = await runFixNestedEmphasis(page, input);
expect(result).toBe(expected);
});
});
test.describe('Full pipeline (fixNestedEmphasis + showdown)', () => {
test('should render nested emphasis as italic with bold inside', async ({ page }) => {
const html = await runFullPipeline(page, '*action *speech* action*');
expect(html).toContain('<em>');
expect(html).toContain('<strong>');
expect(html).toContain('speech');
});
test('should render basic nested example correctly', async ({ page }) => {
const html = await runFullPipeline(page, '*Some Text *Nested Emphasis Text* More text*');
expect(html).toMatch(/<em>.*<strong>Nested Emphasis Text<\/strong>.*<\/em>/);
});
test('should not break normal emphasis rendering', async ({ page }) => {
const html = await runFullPipeline(page, '*normal emphasis*');
expect(html).toContain('<em>normal emphasis</em>');
});
test('should not break bold rendering', async ({ page }) => {
const html = await runFullPipeline(page, '**bold text**');
expect(html).toContain('<strong>bold text</strong>');
});
test('should not break sequential emphasis rendering', async ({ page }) => {
const html = await runFullPipeline(page, '*first* and *second*');
expect(html).toContain('<em>first</em>');
expect(html).toContain('<em>second</em>');
});
test('should render nested bold as single bold block', async ({ page }) => {
const html = await runFullPipeline(page, '**This is **bold**, is it?**');
expect(html).toContain('<strong>');
expect(html).toContain('This is bold');
expect(html).not.toContain('**');
});
test('should render stripped nested bold correctly', async ({ page }) => {
const html = await runFullPipeline(page, '**text **nested** text**');
expect(html).toMatch(/<strong>text nested text<\/strong>/);
});
});
});
/**
* Run fixNestedEmphasis in the browser context.
* @param {import('@playwright/test').Page} page
* @param {string} input
* @returns {Promise<string>}
*/
async function runFixNestedEmphasis(page, input) {
return await page.evaluate(async (text) => {
/** @type {import('../../public/scripts/showdown-emphasis.js')} */
const { fixNestedEmphasis } = await import('./scripts/showdown-emphasis.js');
return fixNestedEmphasis(text);
}, input);
}
/**
* Run fixNestedEmphasis followed by showdown conversion in the browser context.
* @param {import('@playwright/test').Page} page
* @param {string} input
* @returns {Promise<string>}
*/
async function runFullPipeline(page, input) {
return await page.evaluate(async (text) => {
/** @type {import('../../public/scripts/showdown-emphasis.js')} */
const { fixNestedEmphasis } = await import('./scripts/showdown-emphasis.js');
const { showdown } = await import('./lib.js');
const converter = new showdown.Converter({
literalMidWordUnderscores: true,
simpleLineBreaks: true,
});
const fixed = fixNestedEmphasis(text);
return converter.makeHtml(fixed);
}, input);
}