Commit cdf920a6 authored by Benjamin Franzke's avatar Benjamin Franzke
Browse files

[!!!][TASK] Upgrade to CodeMirror v6

CodeMirror v6 is a major rewrite of CodeMirror v5.
Existing addons and language parsers need to be rewritten.
There is a compatibility layer for parsers though, which we
use for our TypoScript parser (which itself is based on
an old CodeMirror javascript parser).

A migration to a grammar based lexer (based on codemirror @lezer
infrastructure) would be desirable in future.

The TypoScript code completion code is adapted via a small shim
that calculates the line-oriented token-state from the
CodeMirror v6 syntax tree. Rewriting the code completer (which
also parses the object structure) has not been an option, without
also rewriting the TypoScript syntax highlighter.
Ideally, both the parser and the code completion would be based
on a modern lezer grammar based parser, allowing to perform
code completion on a proper TypoScript syntax tree.

CodeMirror v6 is authored as ES6 modules and can therefore
be integrated into our JavaScript importmap infrastructure,
without bund...
parent b79f2853
......@@ -120,11 +120,6 @@ module.exports = function (grunt) {
files: {
"<%= paths.workspaces %>Public/Css/preview.css": "<%= paths.sass %>workspace.scss"
}
},
t3editor: {
files: {
'<%= paths.t3editor %>Public/Css/t3editor.css': '<%= paths.sass %>editor.scss'
}
}
},
postcss: {
......@@ -358,10 +353,34 @@ module.exports = function (grunt) {
files: [
{
expand: true,
cwd: '<%= paths.node_modules %>codemirror',
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/codemirror',
src: ['**/*', '!**/src/**', '!rollup.config.js', '!package.json']
}
cwd: '<%= paths.node_modules %>@codemirror',
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/@codemirror/',
rename: (dest, src) => dest + src.replace('/dist/index', ''),
src: ['*/dist/index.js']
},
{
expand: true,
cwd: '<%= paths.node_modules %>@lezer',
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/@lezer/',
rename: (dest, src) => dest + src.replace('/dist/index.es', ''),
src: ['*/dist/index.es.js']
},
{
src: '<%= paths.node_modules %>@lezer/lr/dist/index.js',
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/@lezer/lr.js'
},
{
src: '<%= paths.node_modules %>crelt/index.es.js',
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/crelt.js'
},
{
src: '<%= paths.node_modules %>style-mod/src/style-mod.js',
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/style-mod.js'
},
{
src: '<%= paths.node_modules %>w3c-keyname/index.es.js',
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/w3c-keyname.js'
},
]
}
},
......@@ -690,10 +709,9 @@ module.exports = function (grunt) {
{
expand: true,
src: [
'<%= paths.t3editor %>Public/JavaScript/Contrib/codemirror/**/*.js',
'!<%= paths.t3editor %>Public/JavaScript/Contrib/codemirror/**/*.min.js'
'<%= paths.t3editor %>Public/JavaScript/Contrib/**/*.js'
],
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib/codemirror',
dest: '<%= paths.t3editor %>Public/JavaScript/Contrib',
cwd: '.',
rename: function (dest, src) {
return src;
......
//
// T3editor Styles
// ---------------
// Description: Global styles for t3editor wrapper.
//
@import "../../node_modules/bootstrap/scss/functions";
@import "variables/main";
$panel-bg-color: #f7f7f7;
$panel-border-color: #ddd;
$editor-border-color-changed: #6daadf;
$fullscreen-top: 64px;
$panel-font-size: .85em;
$panel-padding-vertical: 3px;
$panel-padding-horizontal: 6px;
$color-matching-bracket: #6ca52b;
typo3-t3editor-codemirror {
border: 1px solid transparent;
textarea {
width: 100%;
}
* + textarea {
display: none;
}
.CodeMirror-fullscreen {
top: $fullscreen-top !important;
}
.CodeMirror-panel {
background: $panel-bg-color;
padding: $panel-padding-vertical $panel-padding-horizontal;
font-size: $panel-font-size;
&-bottom {
border-top: 1px solid $panel-border-color;
}
}
div.CodeMirror {
span.CodeMirror-matchingbracket {
color: $color-matching-bracket;
}
span.CodeMirror-markText {
background-color: $warning;
}
}
.has-change & {
border-color: $editor-border-color-changed;
}
&[autoheight] .CodeMirror {
height: auto !important;
}
}
......@@ -97,45 +97,49 @@ export function loadModule(payload: JavaScriptItemPayload): Promise<any> {
throw new Error('Unknown JavaScript module type')
}
function executeJavaScriptModuleInstruction(json: JavaScriptItemPayload) {
export function resolveSubjectRef(__esModule: any, payload: JavaScriptItemPayload): any {
const exportName = payload.exportName;
if (typeof exportName === 'string') {
return __esModule[exportName];
}
if ((payload.flags & FLAG_USE_REQUIRE_JS) === FLAG_USE_REQUIRE_JS) {
return __esModule;
}
return __esModule.default;
}
export function executeJavaScriptModuleInstruction(json: JavaScriptItemPayload): Promise<any[]> {
// `name` is required
if (!json.name) {
throw new Error('JavaScript module name is required');
}
if (!json.items) {
loadModule(json);
return;
}
const exportName = json.exportName;
const resolveSubjectRef = (__esModule: any): any => {
if (typeof exportName === 'string') {
return __esModule[exportName];
}
if ((json.flags & FLAG_USE_REQUIRE_JS) === FLAG_USE_REQUIRE_JS) {
return __esModule;
}
return __esModule.default;
return loadModule(json);
}
const items = json.items
.filter((item) => allowedJavaScriptItemTypes.includes(item.type))
.map((item) => {
if (item.type === 'assign') {
return (__esModule: any) => {
const subjectRef = resolveSubjectRef(__esModule);
const subjectRef = resolveSubjectRef(__esModule, json);
mergeRecursive(subjectRef, item.assignments);
};
} else if (item.type === 'invoke') {
return (__esModule: any) => {
const subjectRef = resolveSubjectRef(__esModule);
subjectRef[item.method].apply(subjectRef, item.args);
return (__esModule: any): any => {
const subjectRef = resolveSubjectRef(__esModule, json);
if ('method' in item && item.method) {
return subjectRef[item.method].apply(subjectRef, item.args);
} else {
return subjectRef(...item.args);
}
};
} else if (item.type === 'instance') {
return (__esModule: any) => {
// this `null` is `thisArg` scope of `Function.bind`,
// which will be reset when invoking `new`
const args = [null].concat(item.args);
const subjectRef = resolveSubjectRef(__esModule);
new (subjectRef.bind.apply(subjectRef, args));
const subjectRef = resolveSubjectRef(__esModule, json);
return new (subjectRef.bind.apply(subjectRef, args));
}
} else {
return (__esModule: any) => {
......@@ -144,8 +148,8 @@ function executeJavaScriptModuleInstruction(json: JavaScriptItemPayload) {
}
});
loadModule(json).then(
(subjectRef) => items.forEach((item) => item.call(null, subjectRef))
return loadModule(json).then(
(subjectRef) => items.map((item) => item.call(null, subjectRef))
);
}
......
......@@ -11,9 +11,14 @@
* The TYPO3 project - inspiring people to share!
*/
import {LitElement, html, css, CSSResult} from 'lit';
import {LitElement, html, css} from 'lit';
import {customElement, property, state} from 'lit/decorators';
import {EditorView, ViewUpdate, lineNumbers, highlightSpecialChars, drawSelection, keymap, KeyBinding} from '@codemirror/view';
import {Extension, EditorState} from '@codemirror/state';
import {syntaxHighlighting, defaultHighlightStyle} from '@codemirror/language';
import {defaultKeymap, indentWithTab} from '@codemirror/commands';
import {oneDark} from '@codemirror/theme-one-dark';
import {executeJavaScriptModuleInstruction, loadModule, resolveSubjectRef, JavaScriptItemPayload} from '@typo3/core/java-script-item-processor';
import '@typo3/backend/element/spinner-element'
interface MarkTextPosition {
......@@ -31,38 +36,91 @@ interface MarkText {
*/
@customElement('typo3-t3editor-codemirror')
export class CodeMirrorElement extends LitElement {
@property() mode: string;
@property() label: string;
@property({type: Array}) addons: string[] = ['codemirror/addon/display/panel'];
@property({type: Object}) options: { [key: string]: any[] } = {};
@property({type: Object}) mode: JavaScriptItemPayload = null;
@property({type: Array}) addons: JavaScriptItemPayload[] = [];
@property({type: Array}) keymaps: JavaScriptItemPayload[] = [];
@property({type: Number}) scrollto: number = 0;
@property({type: Object}) marktext: MarkText[] = [];
@property({type: Number}) lineDigits: number = 0;
@property({type: Boolean}) autoheight: boolean = false;
@property({type: Boolean, reflect: true}) autoheight: boolean = false;
@property({type: Boolean}) nolazyload: boolean = false;
@property({type: Boolean}) readonly: boolean = false;
@property({type: Boolean, reflect: true}) fullscreen: boolean = false;
@property({type: String}) label: string;
@property({type: String}) panel: string = 'bottom';
@state() loaded: boolean = false;
@state() editorView: EditorView = null;
static styles = css`
:host {
display: block;
position: relative;
}
typo3-backend-spinner {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
}
:host {
display: flex;
flex-direction: column;
position: relative;
}
:host([fullscreen]) {
position: fixed;
inset: 64px 0 0;
z-index: 9;
}
typo3-backend-spinner {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
}
#codemirror-parent {
min-height: calc(8px + 12px * 1.4 * var(--rows, 18));
}
#codemirror-parent,
.cm-editor {
display: flex;
flex-direction: column;
flex: 1;
max-height: 100%;
}
.cm-scroller {
min-height: 100%;
max-height: calc(100vh - 10rem);
flex: 1;
color-scheme: dark;
}
:host([fullscreen]) .cm-scroller {
min-height: initial;
max-height: 100%;
}
:host([autoheight]) .cm-scroller {
max-height: initial;
}
.panel {
font-size: .85em;
color: #abb2bf;
background: #282c34;
border-style: solid;
border-color: #7d8799;
border-width: 0;
border-top-width: 1px;
padding: .25em .5em;
}
.panel-top {
border-top-width: 0;
border-bottom-width: 1px;
order: -1;
}
`;
render() {
return html`
<slot></slot>
<slot name="codemirror"></slot>
${this.loaded ? '' : html`<typo3-backend-spinner size="large" variant="dark"></typo3-backend-spinner>`}
<div id="codemirror-parent" @keydown=${(e: KeyboardEvent) => this.onKeydown(e)}></div>
${this.label ? html`<div class="panel panel-${this.panel}">${this.label}</div>` : ''}
${this.editorView === null ? html`<typo3-backend-spinner size="large" variant="dark"></typo3-backend-spinner>` : ''}
`;
}
......@@ -88,107 +146,71 @@ export class CodeMirrorElement extends LitElement {
observer.observe(this);
}
private createPanelNode(position: string, label: string): HTMLElement {
const node = document.createElement('div');
node.setAttribute('class', 'CodeMirror-panel CodeMirror-panel-' + position);
node.setAttribute('id', 'panel-' + position);
const span = document.createElement('span');
span.textContent = label;
node.appendChild(span);
return node;
private onKeydown(event: KeyboardEvent): void {
if (event.ctrlKey && event.altKey && event.key === 'f') {
event.preventDefault();
this.fullscreen = true;
}
if (event.key === 'Escape' && this.fullscreen) {
event.preventDefault();
this.fullscreen = false;
}
}
private initializeEditor(textarea: HTMLTextAreaElement): void {
const modeParts = this.mode.split('/');
const options = this.options;
// load mode + registered addons
// @todo: Migrate away from RequireJS usage
window.require(['codemirror', this.mode, ...this.addons], (CodeMirror: typeof import('codemirror')): void => {
const cm = CodeMirror((node: HTMLElement): void => {
const wrapper = document.createElement('div');
wrapper.setAttribute('slot', 'codemirror');
wrapper.appendChild(node);
this.insertBefore(wrapper, textarea);
}, {
value: textarea.value,
extraKeys: {
'Ctrl-F': 'findPersistent',
'Cmd-F': 'findPersistent',
'Ctrl-Alt-F': (codemirror: typeof CodeMirror): void => {
codemirror.setOption('fullScreen', !codemirror.getOption('fullScreen'));
},
'Ctrl-Space': 'autocomplete',
'Esc': (codemirror: typeof CodeMirror): void => {
if (codemirror.getOption('fullScreen')) {
codemirror.setOption('fullScreen', false);
}
},
},
fullScreen: false,
lineNumbers: true,
lineWrapping: true,
mode: modeParts[modeParts.length - 1],
});
// set options
Object.keys(options).map((key: string): void => {
cm.setOption(key, options[key]);
});
// Mark form as changed if code editor content has changed
cm.on('change', (): void => {
textarea.value = cm.getValue();
private async initializeEditor(textarea: HTMLTextAreaElement): Promise<void> {
const updateListener = EditorView.updateListener.of((v: ViewUpdate) => {
if (v.docChanged) {
textarea.value = v.state.doc.toString();
textarea.dispatchEvent(new CustomEvent('change', {bubbles: true}));
});
const panel = this.createPanelNode(this.panel, this.label);
cm.addPanel(
panel,
{
position: this.panel,
stable: false,
},
);
// cm.addPanel() changes the height of the editor, thus we have to override it here again
if (textarea.getAttribute('rows')) {
const lineHeight = 18;
const paddingBottom = 4;
cm.setSize(null, parseInt(textarea.getAttribute('rows'), 10) * lineHeight + paddingBottom + panel.getBoundingClientRect().height);
} else {
// Textarea has no "rows" attribute configured. Set the height to "auto"
// to instruct CodeMirror to automatically resize the editor depending
// on its content.
cm.getWrapperElement().style.height = 'auto';
cm.setOption('viewportMargin', Infinity);
}
});
if (this.autoheight) {
cm.setOption('viewportMargin', Infinity);
}
if (this.lineDigits > 0) {
this.style.setProperty('--rows', this.lineDigits.toString());
} else if (textarea.getAttribute('rows')) {
this.style.setProperty('--rows', textarea.getAttribute('rows'));
}
if (this.lineDigits > 0) {
cm.setOption('lineNumberFormatter', (line: number): string => line.toString().padStart(this.lineDigits, ' '))
}
const extensions: Extension[] = [
oneDark,
updateListener,
lineNumbers(),
highlightSpecialChars(),
drawSelection(),
EditorState.allowMultipleSelections.of(true),
syntaxHighlighting(defaultHighlightStyle, {fallback: true}),
];
if (this.readonly) {
extensions.push(EditorState.readOnly.of(true));
}
if (this.scrollto > 0) {
cm.scrollIntoView({
line: this.scrollto,
ch: 0
});
}
if (this.mode) {
const modeImplementation = <Extension[]>await executeJavaScriptModuleInstruction(this.mode);
extensions.push(...modeImplementation);
}
for (let textblock of this.marktext) {
if (textblock.from && textblock.to) {
cm.markText(textblock.from, textblock.to, {className: 'CodeMirror-markText'});
}
}
if (this.addons.length > 0) {
extensions.push(...await Promise.all(this.addons.map(moduleInstruction => executeJavaScriptModuleInstruction(moduleInstruction))));
}
this.loaded = true;
});
const keymaps: KeyBinding[] = [
...defaultKeymap,
indentWithTab,
];
if (this.keymaps.length > 0) {
const dynamicKeymaps: KeyBinding[][] = await Promise.all(this.keymaps.map(keymap => loadModule(keymap).then((module) => resolveSubjectRef(module, keymap))));
dynamicKeymaps.forEach(keymap => keymaps.push(...keymap));
}
extensions.push(keymap.of(keymaps));
this.editorView = new EditorView({
state: EditorState.create({
doc: textarea.value,
extensions
}),
parent: this.renderRoot.querySelector('#codemirror-parent'),
root: this.renderRoot as ShadowRoot
})
}
}
import {StreamLanguage, LanguageSupport} from '@codemirror/language';
import {CompletionContext, CompletionResult} from '@codemirror/autocomplete';
import {typoScriptStreamParser} from '@typo3/t3editor/stream-parser/typoscript';
import TsCodeCompletion from '@typo3/t3editor/autocomplete/ts-code-completion';
import {syntaxTree} from '@codemirror/language';
import type {SyntaxNodeRef} from '@lezer/common';
interface Token {
type: string;
string: string;
start: number;
end: number;
}
interface CodeMirror5CompatibleCompletionState {
lineTokens: Token[][];
currentLineNumber: number;
currentLine: string;
lineCount: number;
completingAfterDot: boolean,
token?: Token
}
/**
* Module: @typo3/t3editor/language/typoscript
*
* Entry Point module for CodeMirror v6 language highlighting and code completion for TypoScript.
* This module combines our CodeMirror v5 style typoscript parser (via the StreamLanguage shim) and
* our CodeMirror v5 style TypoScript hinter (via an own CodeMirror v5 state shim).
*
* @todo: This is ideally to be replaced by an own CodeMirror v6 style parser/completion logic
* based on lezer.codemirror.net at some point.
*/
export function typoscript() {
const language = StreamLanguage.define(typoScriptStreamParser);
const completion = language.data.of({
autocomplete: complete
})
return new LanguageSupport(language, [completion]);
}
export function complete (context: CompletionContext): Promise<CompletionResult | null> | CompletionResult | null {
let word = context.matchBefore(/\w*/)
if (!context.explicit) {
return null;
}
const cm5state = parseCodeMirror5CompatibleCompletionState(context);
const tokenPos = context.pos - (cm5state.completingAfterDot ? 1 : 0);
const token = syntaxTree(context.state).resolveInner(tokenPos, -1);
const tokenValue = token.name === 'Document' || cm5state.completingAfterDot ? '' : context.state.sliceDoc(token.from, tokenPos);
const completionStart = token.name === 'Document' || cm5state.completingAfterDot ? context.pos : token.from;
let tokenMetadata: Token = {
start: token.from,
end: tokenPos,
string: tokenValue,
type: token.name
};
// If it's not a 'word-style' token, ignore the token.
if (!/^[\w$_]*$/.test(tokenValue)) {
tokenMetadata = {
start: context.pos,
end: context.pos,
string: '',
type: tokenValue === '.' ? 'property' : null
};
}
cm5state.token = tokenMetadata;
const keywords = TsCodeCompletion.refreshCodeCompletion(cm5state);
if ((token.name === 'string' || token.name === 'comment') && tokenIsSubStringOfKeywords(tokenValue, keywords)) {
return null;
}
const completions = getCompletions(tokenValue, keywords);
return {