diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index c288cb87..ff1bc5c9 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -132,7 +132,14 @@ export default defineConfig({ ], }, { text: 'Snippets', link: '/guide/extensibility/snippets.md' }, - { text: 'Syntax Definitions', link: '/guide/extensibility/syntaxdefs.md' }, + { + text: 'Syntax Definitions', + link: '/guide/extensibility/syntax/', + items: [ + { text: 'Tutorial', link: '/guide/extensibility/syntax/tutorial.md' }, + { text: 'Tutorial (Legacy)', link: '/guide/extensibility/syntax/tutorial_legacy.md' }, + ], + }, { text: 'Troubleshooting', link: '/guide/extensibility/troubleshooting.md' }, ], }, @@ -187,8 +194,13 @@ export default defineConfig({ { text: 'Python API', link: '/reference/python_api.md' }, { text: 'Settings', link: '/reference/settings.md' }, { text: 'Symbols', link: '/reference/symbols.md' }, - { text: 'Syntax', link: 'https://www.sublimetext.com/docs/syntax.html' }, - { text: 'Syntax Definitions Legacy', link: '/reference/syntaxdefs_legacy.md' }, + { + text: 'Syntax', + items: [ + { text: 'Syntax Reference', link: 'https://www.sublimetext.com/docs/syntax.html' }, + { text: 'Legacy Syntax Formats', link: '/reference/syntaxdefs_legacy.md' }, + ], + }, { text: 'Keyboard Shortcuts', items: [ diff --git a/docs/guide/extensibility/snippets.md b/docs/guide/extensibility/snippets.md index 9241db0b..55d71c0a 100644 --- a/docs/guide/extensibility/snippets.md +++ b/docs/guide/extensibility/snippets.md @@ -82,7 +82,7 @@ look at each of these parts in turn. : Used when showing the snippet in the Snippets menu. If not present, Sublime Text defaults to the file name of the snippet. -[Scopes]: /guide/extensibility/syntaxdefs.md#scopes +[Scopes]: /guide/extensibility/syntax/index.md#scopes With this information, you can start writing your own snippets as described in the next sections. diff --git a/docs/guide/extensibility/syntax/index.md b/docs/guide/extensibility/syntax/index.md new file mode 100644 index 00000000..552baaf4 --- /dev/null +++ b/docs/guide/extensibility/syntax/index.md @@ -0,0 +1,559 @@ +--- +title: Syntax Definitions +--- + + +# Syntax Definitions + +Syntax definitions make Sublime Text aware +of programming and markup languages. +Most noticeably, they work together with color schemes +to provide syntax highlighting. +Syntax definitions define *[scopes](#scopes)* +that divide the text in a buffer into named regions. +Several editing features in Sublime Text make extensive use +of this fine-grained contextual information. + +Essentially, syntax definitions consist +of regular expressions used to find text, +as well as more-or-less arbitrary, dot-separated strings +called *scopes* or *scope names*. +For every occurrence of a given regular expression, +Sublime Text gives the matching text its corresponding *scope name*. + + +## Syntax Definition Formats + +### `sublime-syntax` + +For Sublime Text 3 (Build 3084), +a new syntax definition format has been added +with the `.sublime-syntax` extension. +This is based on YAML +and uses slightly different keywords. + +It is highly encouraged to be used +in favor of the legacy TextMate format below, +unless compatibility with older versions +or other editors is desired. + +Reference documentation is available +on the [official website][sublime-syntax]. + +[sublime-syntax]: https://www.sublimetext.com/docs/syntax.html + + +### TextMate `tmLanguage` and derivatives + +Sublime Text originally used TextMate language files +(with the `.tmLanguage` extension) +in [property list][plist] (PList) format +for syntax definitions. +Because the XML of PList was cumbersome, +many developers used a YAML or JSON representation +and compiled it to PList afterward. + +Reference documentation is available +at [TextMate Syntax Definitions](/reference/syntaxdefs_legacy.md) + +[plist]: https://en.wikipedia.org/wiki/Property_list + + +## Scopes + +::: info See Also +[Scope Naming][] +: Official documentation on assigning scopes to code, + including a section on Color Schemes + +[Selectors][scope selectors] +: Official documentation on scope selectors +::: + +Scopes are a key concept in Sublime Text, +which it inherits from the macOS editor TextMate. +Essentially, scopes are named text regions in a buffer. +They don't do anything by themselves, +but Sublime Text peeks at them when it needs contextual information. + +For instance, when you trigger a snippet, Sublime Text checks the scope +bound to the snippet and looks at the caret's position in the file. If +the caret's current position matches the snippet's scope selector, +Sublime Text fires it off. Otherwise, nothing happens. + +Furthermore, [Color Schemes][] make extensive use of scopes +to style every aspect of a language in the desired color. + +::: tip Info +There's a slight difference between *scopes* and *[scope selectors][]*: Scopes +are the names defined in a syntax definition, while scope selectors are used +in items like snippets and key bindings to target scopes. When creating a +new syntax definition, you care about scopes; when you want to constrain a +snippet to a certain scope, you use a scope selector. +::: + +Scopes can be nested to allow for a high degree of granularity. You can drill +down the hierarchy very much like with CSS selectors. For instance, thanks to +scope selectors, you could have a key binding activated only within single +quoted strings in Python source code, but not inside single quoted strings in +any other language. + +[scope naming]: https://www.sublimetext.com/docs/scope_naming.html +[scope selectors]: https://www.sublimetext.com/docs/selectors.html +[color schemes]: /guide/customization/color_schemes.md + + +## How Syntax Definitions Work + +At their core, +syntax definitions are arrays of regular expressions +paired with scope names. +Sublime Text will try to match these patterns +against a buffer's text +and attach the corresponding scope name +to each occurrence. +These pairs of regular expressions and scope names +are known as *rules*. +Sets of rules called *contexts* +are pushed to and popped from a stack. + +### Basic Processing + +The basic loop for the syntax engine looks like this: + +1. The engine pushes the first context, `main`. + It also sets the current character + to the first character + of the unprocessed editor text. + +2. From the current character + to the end of its line, + evaluate the regular expressions + in the current stack frame + from first to last + until one matches. + +3. Consume characters + in the matching regexp + and assign scopes to them. + +4. Optionally push or pop a context on the stack. + The `sublime-syntax` format has a significant change + from TextMate, + wherein multiple stack frames can be pushed or popped at a time. + It also supports replacing the current context + with a different one. + +5. If none of the regular expressions + in the current context match + at the current position in the file, + the engine will advance + to the next character of the file + and restart the list of regexps + in the current context. + +6. Go to step 2 at the new character position. + +::: warning Caveats +- Regular expressions that consume no characters + should change the stack. + If they do not, + the current character is advanced + as in step 5 + to avoid an infinite loop. + Common examples are + lookaheads like `(?=\S)`, + BOL or EOL anchors like `^` and `$`, + and the null regexp `''` that always matches. + +- Regular expressions *do not match* across line breaks. + +- Since rules are matched in order, + make sure that more specific rules + come sooner in each context. + Otherwise, a greedy regular expression might swallow parts + you'd like to have styled differently. +::: + + +## Reusing Matches + +### `contexts` + +It would be a mess to repeat relevant regular expressions +in each stack frame that needed them. +This is the purpose of the `contexts` section. +Contexts are sets of matches and other instructions +that a stack frame can `include` in its matches. +They are processed in the same order +as if they were `match` instructions +at the location of the `include` instruction. + +Contexts can also be pushed onto the stack +as a new frame. +This is why stack frames are often +colloquially referred to as "contexts," +including in Sublime Text's scope debugger. + +::: tip +- Contexts can be included from separate files. + +- Rules can recurse through pushed contexts. + (`main` -> `brace-blocks` -> `main`) +::: + +### `variables` + +Portions of regular expressions can also be saved +in the `variables` section and reused in multiple expressions. +Variables also make some ugly regexps readable. + + +### Match reuse best practice + +Well-designed syntax definitions will define utility contexts +that `include` equivalent things together for re-usability: + +- A normal programming language will have things like + - A **statements** group of all things that can be directly executed. + This then may or may not (language-dependent) include… + - An **expressions** group of things + that you can put on the right-hand-side of an assignment, + which will definitely include… + - An **atoms** group of strings, numbers, chars, etc. + that might also be valid statements, + but that also depends on your language. + - **function-definitions** probably won't be in **expressions** + (unless they are lambdas) + but probably _would_ be in **statements**. + Function definitions might push into a context + that lets you `return` and so on. + +- A markup language might have + - An **inlines** group to keep track of all the markup + one can have within a block. + - A **blocks** group to hold lists, quotes, paragraphs, headers. + - … + + +## Manipulating the Stack + +Sublime's syntax definitions allow better stack control +than TextMate languages, +allowing some common patterns. +These are adapted from [the Tips issue][tips] +on the repository for ST's own syntaxes. + +[tips]: https://github.com/sublimehq/Packages/issues/757 + + +### Pushing multiple contexts + +When you have a construction +where you expect a list of elements in sequence, +put them all onto the stack at once. +The stack will unwind as the elements are recognized. + +```yaml +contexts: + + else-pop: + - match: (?=\S) + pop: 1 + + functions: + - match: function(?=\s) + scope: keyword.declaration.function + push: + - function-body + - function-params + - function-name + + function-name: + - match: (?:{{identifier_function}})?(?=[({]) + scope: entity.name.function + pop: 1 + - include: storage-modifiers # global, private, etc. + + function-params: + - match: \( + scope: punctuation.section.parameters.begin + push: function-param-body + - include: else-pop + + function-param-body: + - meta_scope: meta.function.parameters + - match: \) + scope: punctuation.section.parameters.end + pop: 2 + - ... + + function-body: + - meta_scope: meta.function + - match: \{ + scope: punctuation.section.block.begin + push: function-body-content + - include: else-pop + + function-body-content: + - match: \} + scope: punctuation.section.block.end + pop: 2 + - include: statements +``` + +As an added benefit, most of these scopes can be reused: + +```yaml + immediately-pop: + - match: '' + pop: 1 + + statements: + ... + - match: \{ + scope: punctuation.section.braces.begin + push: + - meta-block + - expect-closing-brace + - statements + ... + + meta-block: + - meta_scope: meta.block + - include: immediately-pop +``` + +As a bonus, states stacked this way are implicitly optional. +If one is omitted, +the highlighter will move on to the next without interruption. +For instance, in the first example, +the construction will be parsed correctly +whether or not the author supplies a function name. + +::: tip Tip +Use plural context names to indicate non-popping contexts. +In other words, plural contexts can match multiple times. + +Use singular context names where the contents can only match once. +::: + + +### Context chaining + +You can also manipulate the stack with sequences of `set`s. +Before making an elaborate state machine, +ask yourself if you *really* need to. + + +#### Push your first state + +While it is absolutely possible to have a match in `main` +which `set`s into a chain of stateful contexts +and subsequently sets back into `main` at the end, +it is not recommended. +`main` should be a stateless "baseline" context +that is always the last element on the stack. + +Instead, have your match in `main` use push +to get into your first state, +then `pop` out of the last state. +For example, imagine we wanted +to match the sequence `abc` with each character scoped differently +and only when they follow each other. +For illustration purposes, we will also match numerics in `main`: + +```yaml +contexts: + main: + - match: a + scope: first + push: expect-b + - match: \d+ + scope: constant.numeric + + expect-b: + - match: b + scope: second + set: expect-c + + expect-c: + - match: c + scope: third + pop: 1 +``` + +Notice how `a` pushes `expect-b`. +We don't set the first context, only the second one. +Once we find the terminator, we pop out. + + +#### Lookahead push for meta scoping + +Sometimes you need to apply a meta scope +to an entire stateful chunk. +When this is the case, +you almost certainly want your push rule +to be a non-consuming lookahead +rather than a consuming scoped match. +We can modify the above: + +```yaml +contexts: + main: + - match: (?=a) + push: expect-a + - match: \d+ + scope: constant.numeric + + expect-a: + - meta_scope: meta.abc + - match: a + scope: first + set: expect-b + + expect-b: + - meta_scope: meta.abc + - match: b + scope: second + set: expect-c + + expect-c: + - meta_scope: meta.abc + - match: c + scope: third + pop: 1 +``` + + +#### Bail outs + +Always remember that you're writing a parser +for a set of partially valid syntax fragments. +The normal mode of operation is that someone is actively typing new text. +For this reason, +make sure that any and all stateful contexts you use +have aggressive "bail-outs" for when something goes wrong. +As a rule of thumb, if there's a case where a compiler's parser would have produced an error, +your syntax mode should handle that case by `pop`ing back to `main`. + +Consider the example from above. Imagine the user is typing typing into the following buffer: + +``` +42 +ab +12 +``` + +Even if the user is actively typing `c` following `b`, +it would be a terrible experience for the scoping on `12` +to shift back and forth as they type in the middle. +For this reason, you should always end your mid-state scopes +with a lookahead match like `else-pop` from the multi-push section above +that pops out of the state chain. + +Getting this wrong is one of the easiest ways +to create a terrible experience for users of your mode +without even realizing it yourself. + + +## Other Instruction Keywords + +Consult [the official documentation][sublime-syntax] +for more detail on the terms below. + + +### Meta scopes + +`meta_scope` +: Apply a scope to a whole context, + including the matches that push and pop it. + +`meta_content_scope` +: Apply a scope to a whole context, + except for the matches that push and pop it. + + +### Prototyping + +The `prototype` context +: A special context included + at the beginning of every context + except contexts included by `prototype` itself. + For example: comments + +`meta_include_prototype` +: Keyword to disable including `prototype`. + For example: inside strings + +`with_prototype` +: When pushing a context, + also include these rules + at the beginning of *every* nested context. + + +### Embedding + +`embed` +: Like a pushed context, + popping rules are different + as described in `escape`. + +`escape` +: Aggressively return directly to the embedding context, + popping any number of contexts upon match. + +`escape_captures` +: Allow assigning scopes to the regexp in `escape`. + + +### Branching + +Sometimes the appropriate scope is not decidable +without context beyond a line break. +For these cases, the `branch` keyword +describes an array of speculative contexts +to try until a `fail` match rewinds +back to the `branch_point`. + + +### Inheritance + +Syntaxes can extend from other syntaxes. +Each variable can be overridden at will. +Each context can be prepended to, appended to, or replaced outright. + + +## Regular Expression Performance + +Sublime built a custom regexp engine to process rules, +commonly called `sregex`. +It explicitly excludes support for certain constructs +that are slow or explode backtracking. + +Oniguruma is still available and used where necessary, +but the best practice for development is to eliminate incompatible patterns. + +In practice, this means to avoid + +- Anything non-regular in the formal sense + (backreferences, recursive matches, etc.). + + Except when capture groups are used in a `push`. + Those are available in a `pop` as backrefs. + +- Lookbehinds + + Except in `escape` patterns. + +- Atomic groups and possessive quantifiers + +- Some Unicode character properties + +- Named captures + + +### Testing `sregex` Compatibility + +Syntax definitions have a build variant +to test pattern compatibility with the `sregex` engine. +Use and choose the +**Syntax Tests - Regex Compatibility** option. diff --git a/docs/guide/extensibility/syntax/tutorial.md b/docs/guide/extensibility/syntax/tutorial.md new file mode 100644 index 00000000..d3761dd9 --- /dev/null +++ b/docs/guide/extensibility/syntax/tutorial.md @@ -0,0 +1,586 @@ +--- +title: Syntax Definition Tutorial +--- + +# Syntax Definitions + +## Prerequisites + +In order to follow this tutorial, +you will need to install [PackageDev][], +a package intended to ease the creation +of new syntax definitions for Sublime Text. +Follow the installation notes +in the "Getting Started" section of its ReadMe. + +[packagedev]: https://github.com/SublimeText/PackageDev + + +## Your First Syntax Definition + +By way of example, let's create a syntax definition for Sublime Text +snippets. We'll be styling the actual snippet content, not the whole +`.sublime-snippet` file. + +::: tip Note +Since syntax definitions are primarily used to enable syntax highlighting, +we'll use the phrase *to style* to mean *to break down a source code file +into scopes*. Keep in mind, however, that colors are a different thing from +syntax definitions and that scopes have many more uses besides syntax +highlighting. +::: + +Here are the elements we want to style in a snippet: + +- Variables (`$PARAM1`, `$USER_NAME`\ ...) +- Simple fields (`$0`, `$1`\ ...) +- Complex fields with placeholders (`${1:Hello}`) +- Nested fields (`${1:Hello ${2:World}!}`) +- Escape sequences (`\$`, `\<`, …) +- Illegal sequences (`$`, `<`, `\`, …) + +Here are the elements we don't want to style because they are too complex for +this example: + +- Variable Substitution (`${1/Hello/Hi/g}`) + +::: tip Note +Before continuing, make sure you've installed the + package as explained above. +::: + + +## Creating a New Syntax Definition + +To create a new syntax definition, follow these steps: + +1. Open the Command Palette and choose **New Syntax…** +1. Save the new file in your `Packages/User` folder + as a `.sublime-syntax` file. + +You now should see a file like this: + +```yaml +%YAML 1.2 +--- +# See https://www.sublimetext.com/docs/syntax.html +file_extensions: + - ec +scope: source.example-c +contexts: + main: +# ... +``` + +Let's examine the key elements. + +`scope` +: The topmost [scope][] for this syntax definition. + It takes the form `source.` or `text.`. + For programming languages, use `source`. + For markup and everything else, use `text`. + +`file_extensions` +: This is a list of file extensions (without the leading dot). + When opening files of these types, + Sublime Text will automatically activate this syntax definition for them. + +`contexts` +: A container for contexts that contain your match patterns. + You'll notice the `main` context + from [How Syntax Definitions Work][syndef-work] + and some other sample contexts we won't need. + +[scope]: index.md#scopes +[syndef-work]: index.md#how-syntax-definitions-work + +For our example, edit the template with the following information, +and throw away everything after `main:`. + +```yaml +name: Sublime Snippet (Raw) +scope: source.ssraw +file_extensions: + - ssraw + +contexts: + main: +``` + +::: tip Note +YAML is not a very strict format, but can cause headaches when you don't +know its conventions. It supports single and double quotes, but you may also +omit them as long as the content does not create another YAML literal. + + syntax highlighting is very good +at demonstrating where strings will be correctly or incorrectly parsed. +When quotes are necessary, convention is to use single quotes. + +The `---` and `...` are optional. +::: + + +## Creating some rules + +If you don't remember [how syntax definition processing works][syndef-work] +at a general level, +you may want to read [a synopsis][syndef-work]. + +Values in the `contexts` dictionary can contain several types of element. +We'll look at some of them in the following sections. +If you want to learn more, +refer to [the official documentation][sublime-syntax]. + + +### Matches + +Matches take this form: + +```yaml +- match: (?i:m)y \s+[Rr]egex + scope: string.format +``` + +Sublime Text uses a custom engine called `sregex` +for regular expressions in syntax definitions +that uses [Oniguruma][]'s format. +There is a fallback to the Oniguruma engine +for features `sregex` doesn't support. +Several existing syntax definitions make use of +features supported by Oniguruma that aren't part of +PERL-style regular expressions, hence the requirement for the fallback. + +[oniguruma]: https://github.com/kkos/oniguruma/blob/master/doc/RE + +`match` +: A regular expression Sublime Text will use to find matches. + +`scope` +: The name of the [scope][] that should be applied + to any occurrences of `match`. + +Let's go back to our example +and begin to add our rules for Sublime snippets. +We'll start with simple fields. +These could be matched with a regex like so: + +```perl +\$[0-9]+ +# or... +\$\d+ +``` + +We can then build our pattern like this: +```yaml +# Tab stops like $1, $2... +- match: \$\d+ + scope: variable.language.ssraw +``` + +or even add a capture group to further scope the punctuation: +```yaml +# Tab stops like $1, $2... +- match: (\$)\d+ + scope: variable.language.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw +``` + +Captures introduce complexity to your rule, +but they are pretty straightforward. +Notice how numbers refer to parenthesized groups left to right. +Of course, you can have as many capture groups as you want. + +::: tip Note +As with usual regular expressions and substitutions, the capture group +`0` applies to the whole match. +::: + +We choose `variable.language` because the `$1` is a variable, +but its value is set by the language itself, +not by an assignment or declaration. +The `punctuation...` scope lets color schemes carefully target +the punctuation if their designers want them to. + + +::: tip Choosing the Right Scope Name +Naming scopes isn't obvious sometimes. +Check the [naming conventions][] for guidance on scope names. + automatically provides completions +for scope names according to these conventions. +It is important to re-use the basic categories outlined there +if you want to achieve the highest compatibility +with existing [color schemes][]. + +Color schemes have hardcoded scope names in them. They could not +possibly include every scope name you can think of, so they target the +standard ones plus some rarer ones on occasion (like for CSS or +Markdown). This means that two color schemes using the same syntax +definition may render the text differently! +That's left to the color scheme, though. +Resist any temptation to use novel scopes +to select a specific color in your color scheme. +::: + +[color schemes]: /guide/customization/color_schemes.md +[naming conventions]: https://www.sublimetext.com/docs/scope_naming.html + +And we can add it to our syntax definition too: + +```yaml +name: Sublime Snippet (Raw) +scope: source.ssraw +file_extensions: + - ssraw + +contexts: + main: + # Tab stops like $1, $2... + - match: (\$)\d+ + scope: variable.language.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw +``` + +::: tip Note +You should use two spaces for indent. This is the recommended indent for +YAML and lines up with lists like shown above. +::: + +You have now created your first syntax definition. +Next, open a new file and +save it with the extension `.ssraw`. +The buffer's syntax name should switch +to "Sublime Snippet (Raw)" automatically, +and you should get syntax highlighting +if you type `$1` or any other simple snippet field. + + +### Defining Variables for Regexes + +Let's proceed to creating another rule for environment variables. + +```yaml +# Variables like $SELECTION, $TM_FILEPATH... +- match: (\$)[A-Za-z][A-Za-z0-9_]* + scope: variable.language.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw +``` + +That doesn't quite match what users' experience will be, though: +not every word is a language-supported placeholder. +Let's demote the generic case to a generic variable scope +and add an allowlist of the specific words that snippets recognize. +Be sure to add the special rule before the catch-all. + +```yaml +# Known variables like $SELECTION, $TM_FILEPATH... +- match: (\$)(?:SELECTION|TM_SELECTED_TEXT|TM_LINE_INDEX|this gets very long) + scope: variable.language.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw + +# Unknown variables or incompletely-typed ones +- match: (\$)[A-Za-z][A-Za-z0-9_]* + scope: variable.other.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw +``` + +That list of variables is actually rather long, +and we might even want to use it elsewhere in our file. +Let's pull it out into `variables` and refer to it by name: + +```yaml +contexts: + main: + # ... + + # Known variables like $SELECTION, $TM_FILEPATH... + - match: (\$){{variable_name}} + scope: variable.language.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw + + # Unknown variables or incompletely-typed ones + - match: (\$)[A-Za-z][A-Za-z0-9_]* + scope: variable.other.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw + + # ... + +variables: + variable_name: |- + (?x:\b(?: + SELECTION + | TM_SELECTED_TEXT + | TM_LINE_INDEX + | TM_LINE_NUMBER + | TM_DIRECTORY + | TM_FILEPATH + | TM_FILENAME + | TM_CURRENT_WORD + | TM_CURRENT_LINE + | TM_TAB_SIZE + | TM_SOFT_TABS + | TM_SCOPE + )\b) +``` + +Now `variable_name` holds a regex snippet that can be re-used +by `match` patterns with `{{variable_name}}`. + + +::: tip Best Practice +Remember that variables are stamped directly +into the regex string as parsed by YAML. + +Avoid capturing groups if possible. +They will offset the `captures` in a match. + +But wrap your variables in a non-capturing group! +This lets the match patterns apply quantifiers. +If you assign variable **boolean** to `true|false`, +then `{{boolean}}?` will only make the `e` in "false" optional. +You want `(?:true|false)` instead. + +Long variables (or `match` regexes) can use multi-line mode +for ease of comprehension, +usually with a YAML block string. +You can even leave line comments with `#`. +::: + + +### Push and Pop Rules + +Up to now we've been using a simple rule. +Although we've seen how to dissect patterns into smaller components, +sometimes you'll want to target a larger portion of your source code +that is clearly delimited by start and end marks. + +From [the synopsis of how syntax engines work][syndef-work], +you'll remember that the array of rules to match on +can be changed by manipulating the stack of [contexts](index.md#contexts). +Literal strings enclosed by quotation marks or other delimiting constructs +are better dealt with by push and pop rules. +This is a skeleton for one of these rules +that pushes an anonymous context, +namely one that doesn't have an identifier in `contexts`. + +```yaml +- match: + scope: + push: + - match: + scope: + pop: 1 +``` + +That is the simplest version. +Let's take a look at one that includes all available options: + +``` yaml +- match: + scope: + push: + - meta_scope: + - meta_content_scope: + - match: + scope: + pop: 1 +# ... +``` + +Some elements may look familiar, but their combination might be +daunting. Let's inspect them individually. + +`meta_scope` +: This sets the following scope name to + *the whole context* and the match that pushed it, + above any of that match's `scope` or `captures`. + **Optional.** + +`meta_content_scope` +: Unlike the `meta_scope`, + this only applies a scope name to the portion of the context + that does not include the pushing or popping matches. + **Optional.** + +outer `match` +: Regex for the opening mark for this scope. + +inner `match` +: Regex for the end mark for this scope. + +`pop` +: Notes a number of contexts to pop off of the stack. + Multiple matches can have `pop` instructions. + **Optional.** + +We'll use this rule to style nested complex fields in snippets: + +```yaml +- match: (\$)(\{)[0-9]+(:) + captures: + 1: punctuation.definition.variable.begin.ssraw + 2: punctuation.section.interpolation.begin.ssraw + 3: punctuation.separator.ssraw + push: + - meta_scope: variable.language.complex.ssraw + - meta_content_scope: string.unquoted.ssraw + - match: \} + scope: punctuation.section.interpolation.end.ssraw + pop: 1 + - include: main +``` + +Although it is possible to push anonymous contexts, +best practice is to name them for ease of debugging. + +Let's give this context a name: + +```yaml +contexts: + main: + # ... (other rules) ... + + # Complex variables ${: ... } + - match: (\$)(\{)[0-9]+(:) + captures: + 1: punctuation.definition.variable.begin.ssraw + 2: punctuation.section.interpolation.begin.ssraw + 3: punctuation.separator.ssraw + push: complex_variable_body + + complex_variable_body: + - meta_scope: variable.language.complex.ssraw + - meta_content_scope: string.unquoted.ssraw + - match: \} + scope: punctuation.section.interpolation.end.ssraw + pop: 1 + - include: main +``` + +This is the most complex pattern we'll see in this tutorial. + +Notice that there are other matches and includes +that do not pop the context stack. +These will be matched until a `}` is encountered. +It even includes **the `main` context,** +which happily recurses if another `${\d` match is discovered! + +Remember, [matched text is consumed][syndef-work]. +It is consequently excluded from the next match attempt +and can't be matched again. +Make sure your additional matches **do not** +accidentally eat the popping match. + +To finish off complex fields, +we've styled the placeholders as strings +with the `meta_content_scope` field. + + +### Final Touches + +Lastly, let's style escape sequences and illegal sequences, and then we +can wrap up. + +```yaml +# Sequences like \$, \> and \< +- match: \\[$<>] + scope: constant.character.escape.ssraw + +# Unescaped and unmatched magic characters +- match: '[$<>]' + scope: invalid.illegal.ssraw +``` + +The only hard thing here is not forgetting that `[]` enclose arrays in +YAML and thus must be wrapped in quotes. Other than that, the rules are +pretty straightforward if you're familiar with regular expressions. + +However, you must take care to place the second rule after any others +matching the `$` character, since otherwise it will be consumed and +result in every following expression not matching. + +Also, even after adding these two additional rules, note that our +recursive begin-end rule from above continues to work as expected. + +At long last, here's the final syntax definition: + +```yaml +%YAML 1.2 +--- +# See https://www.sublimetext.com/docs/syntax.html +name: Sublime Snippet (Raw) +scope: source.ssraw +file_extensions: + - ssraw + +contexts: + main: + # Tab stops like $1, $2... + - match: (\$)\d+ + scope: variable.language.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw + + # Known variables like $SELECTION, $TM_FILEPATH... + - match: (\$){{variable_name}} + scope: variable.language.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw + + # Unknown variables or incompletely-typed ones + - match: (\$)[A-Za-z][A-Za-z0-9_]* + scope: variable.other.ssraw + captures: + 1: punctuation.definition.variable.begin.ssraw + + # Complex variables ${: ... } + - match: (\$)(\{)[0-9]+(:) + captures: + 1: punctuation.definition.variable.begin.ssraw + 2: punctuation.section.interpolation.begin.ssraw + 3: punctuation.separator.ssraw + push: complex_variable_body + + # Sequences like \$, \> and \< + - match: \\[$<>] + scope: constant.character.escape.ssraw + + # Unescaped and unmatched magic characters + - match: '[$<>]' + scope: invalid.illegal.ssraw + + complex_variable_body: + - meta_scope: variable.language.complex.ssraw + - meta_content_scope: string.unquoted.ssraw + - match: \} + scope: punctuation.section.interpolation.end.ssraw + pop: 1 + - include: main + +variables: + variable_name: |- + (?x:\b(?: + SELECTION + | TM_SELECTED_TEXT + | TM_LINE_INDEX + | TM_LINE_NUMBER + | TM_DIRECTORY + | TM_FILEPATH + | TM_FILENAME + | TM_CURRENT_WORD + | TM_CURRENT_LINE + | TM_TAB_SIZE + | TM_SOFT_TABS + | TM_SCOPE + )\b) +``` + +There are more available constructs and code reuse techniques, +but the above explanations should get you started with the +creation of syntax definitions. diff --git a/docs/guide/extensibility/syntaxdefs.md b/docs/guide/extensibility/syntax/tutorial_legacy.md similarity index 86% rename from docs/guide/extensibility/syntaxdefs.md rename to docs/guide/extensibility/syntax/tutorial_legacy.md index 8b2321fe..baec0571 100644 --- a/docs/guide/extensibility/syntaxdefs.md +++ b/docs/guide/extensibility/syntax/tutorial_legacy.md @@ -1,8 +1,8 @@ --- -title: Syntax Definitions +title: Syntax Definition Tutorial (Legacy) --- -# Syntax Definitions +# Syntax Definition Tutorial (Legacy) Syntax definitions make Sublime Text aware of programming and markup languages. Most noticeably, they work together with colors to provide syntax highlighting. @@ -65,65 +65,6 @@ XML, but always keep in mind their differing needs in regards to escape sequences, many XML tags etc. -## Scopes - -Scopes are a key concept in Sublime Text. Essentially, they are named -text regions in a buffer. They don't do anything by themselves, but -Sublime Text peeks at them when it needs contextual information. - -For instance, when you trigger a snippet, Sublime Text checks the scope -bound to the snippet and looks at the caret's position in the file. If -the caret's current position matches the snippet's scope selector, -Sublime Text fires it off. Otherwise, nothing happens. - -::: tip Info -There's a slight difference between *scopes* and *scope selectors*: Scopes -are the names defined in a syntax definition, while scope selectors are used -in items like snippets and key bindings to target scopes. When creating a -new syntax definition, you care about scopes; when you want to constrain a -snippet to a certain scope, you use a scope selector. -::: - -Scopes can be nested to allow for a high degree of granularity. You can drill -down the hierarchy very much like with CSS selectors. For instance, thanks to -scope selectors, you could have a key binding activated only within single -quoted strings in Python source code, but not inside single quoted strings in -any other language. - -Sublime Text inherits the idea of scopes from Textmate, a text editor for Mac. -[Textmate's online manual][] contains further information about scope selectors -that's useful for Sublime Text users too. In particular, Color Schemes make -extensive use of scopes to style every aspect of a language in the desired -color. - -[Textmate's online manual]: https://manual.macromates.com/en/scope_selectors - - -## How Syntax Definitions Work - -At their core, syntax definitions are arrays of regular expressions -paired with scope names. Sublime Text will try to match these patterns -against a buffer's text and attach the corresponding scope name to all -occurrences. These pairs of regular expressions and scope names are -known as *rules*. - -Rules are applied in order, one line at a time. Rules are applied in the -following order: - -1. The rule that matches at the first position in a line -2. The rule that comes first in the array - -Each rule consumes the matched text region, which therefore will be -excluded from the next rule's matching attempt (save for a few -exceptions). In practical terms, this means that you should take care to -go from more specific rules to more general ones when you create a new -syntax definition. Otherwise, a greedy regular expression might swallow -parts you'd like to have styled differently. - -Syntax definitions from separate files can be combined, and they can be -recursively applied too. - - ## Your First Syntax Definition By way of example, let's create a syntax definition for Sublime Text @@ -407,7 +348,7 @@ Arguably, you'd want the other scope to be visually consistent with this one. Go ahead and change it too. ::: tip Note -As with ususal regular expressions and substitutions, the capture group +As with usual regular expressions and substitutions, the capture group `'0'` applies to the whole match. ::: diff --git a/docs/reference/completions.md b/docs/reference/completions.md index 9e3ac752..293ac2d6 100644 --- a/docs/reference/completions.md +++ b/docs/reference/completions.md @@ -52,7 +52,7 @@ Here's an example (with HTML completions): See [Scopes][] for more information. -[Scopes]: /guide/extensibility/syntaxdefs.md#scopes +[Scopes]: /guide/extensibility/syntax/index.md#scopes **completions** : Array of *completions*.