diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts
index c288cb87..ff1bc5c9 100644
--- a/docs/.vitepress/config.ts
+++ b/docs/.vitepress/config.ts
@@ -132,7 +132,14 @@ export default defineConfig({
],
},
{ text: 'Snippets', link: '/guide/extensibility/snippets.md' },
- { text: 'Syntax Definitions', link: '/guide/extensibility/syntaxdefs.md' },
+ {
+ text: 'Syntax Definitions',
+ link: '/guide/extensibility/syntax/',
+ items: [
+ { text: 'Tutorial', link: '/guide/extensibility/syntax/tutorial.md' },
+ { text: 'Tutorial (Legacy)', link: '/guide/extensibility/syntax/tutorial_legacy.md' },
+ ],
+ },
{ text: 'Troubleshooting', link: '/guide/extensibility/troubleshooting.md' },
],
},
@@ -187,8 +194,13 @@ export default defineConfig({
{ text: 'Python API', link: '/reference/python_api.md' },
{ text: 'Settings', link: '/reference/settings.md' },
{ text: 'Symbols', link: '/reference/symbols.md' },
- { text: 'Syntax', link: 'https://www.sublimetext.com/docs/syntax.html' },
- { text: 'Syntax Definitions Legacy', link: '/reference/syntaxdefs_legacy.md' },
+ {
+ text: 'Syntax',
+ items: [
+ { text: 'Syntax Reference', link: 'https://www.sublimetext.com/docs/syntax.html' },
+ { text: 'Legacy Syntax Formats', link: '/reference/syntaxdefs_legacy.md' },
+ ],
+ },
{
text: 'Keyboard Shortcuts',
items: [
diff --git a/docs/guide/extensibility/snippets.md b/docs/guide/extensibility/snippets.md
index 9241db0b..55d71c0a 100644
--- a/docs/guide/extensibility/snippets.md
+++ b/docs/guide/extensibility/snippets.md
@@ -82,7 +82,7 @@ look at each of these parts in turn.
: Used when showing the snippet in the Snippets menu. If not present, Sublime
Text defaults to the file name of the snippet.
-[Scopes]: /guide/extensibility/syntaxdefs.md#scopes
+[Scopes]: /guide/extensibility/syntax/index.md#scopes
With this information, you can start writing your own snippets as described in
the next sections.
diff --git a/docs/guide/extensibility/syntax/index.md b/docs/guide/extensibility/syntax/index.md
new file mode 100644
index 00000000..552baaf4
--- /dev/null
+++ b/docs/guide/extensibility/syntax/index.md
@@ -0,0 +1,559 @@
+---
+title: Syntax Definitions
+---
+
+
+# Syntax Definitions
+
+Syntax definitions make Sublime Text aware
+of programming and markup languages.
+Most noticeably, they work together with color schemes
+to provide syntax highlighting.
+Syntax definitions define *[scopes](#scopes)*
+that divide the text in a buffer into named regions.
+Several editing features in Sublime Text make extensive use
+of this fine-grained contextual information.
+
+Essentially, syntax definitions consist
+of regular expressions used to find text,
+as well as more-or-less arbitrary, dot-separated strings
+called *scopes* or *scope names*.
+For every occurrence of a given regular expression,
+Sublime Text gives the matching text its corresponding *scope name*.
+
+
+## Syntax Definition Formats
+
+### `sublime-syntax`
+
+For Sublime Text 3 (Build 3084),
+a new syntax definition format has been added
+with the `.sublime-syntax` extension.
+This is based on YAML
+and uses slightly different keywords.
+
+It is highly encouraged to be used
+in favor of the legacy TextMate format below,
+unless compatibility with older versions
+or other editors is desired.
+
+Reference documentation is available
+on the [official website][sublime-syntax].
+
+[sublime-syntax]: https://www.sublimetext.com/docs/syntax.html
+
+
+### TextMate `tmLanguage` and derivatives
+
+Sublime Text originally used TextMate language files
+(with the `.tmLanguage` extension)
+in [property list][plist] (PList) format
+for syntax definitions.
+Because the XML of PList was cumbersome,
+many developers used a YAML or JSON representation
+and compiled it to PList afterward.
+
+Reference documentation is available
+at [TextMate Syntax Definitions](/reference/syntaxdefs_legacy.md)
+
+[plist]: https://en.wikipedia.org/wiki/Property_list
+
+
+## Scopes
+
+::: info See Also
+[Scope Naming][]
+: Official documentation on assigning scopes to code,
+ including a section on Color Schemes
+
+[Selectors][scope selectors]
+: Official documentation on scope selectors
+:::
+
+Scopes are a key concept in Sublime Text,
+which it inherits from the macOS editor TextMate.
+Essentially, scopes are named text regions in a buffer.
+They don't do anything by themselves,
+but Sublime Text peeks at them when it needs contextual information.
+
+For instance, when you trigger a snippet, Sublime Text checks the scope
+bound to the snippet and looks at the caret's position in the file. If
+the caret's current position matches the snippet's scope selector,
+Sublime Text fires it off. Otherwise, nothing happens.
+
+Furthermore, [Color Schemes][] make extensive use of scopes
+to style every aspect of a language in the desired color.
+
+::: tip Info
+There's a slight difference between *scopes* and *[scope selectors][]*: Scopes
+are the names defined in a syntax definition, while scope selectors are used
+in items like snippets and key bindings to target scopes. When creating a
+new syntax definition, you care about scopes; when you want to constrain a
+snippet to a certain scope, you use a scope selector.
+:::
+
+Scopes can be nested to allow for a high degree of granularity. You can drill
+down the hierarchy very much like with CSS selectors. For instance, thanks to
+scope selectors, you could have a key binding activated only within single
+quoted strings in Python source code, but not inside single quoted strings in
+any other language.
+
+[scope naming]: https://www.sublimetext.com/docs/scope_naming.html
+[scope selectors]: https://www.sublimetext.com/docs/selectors.html
+[color schemes]: /guide/customization/color_schemes.md
+
+
+## How Syntax Definitions Work
+
+At their core,
+syntax definitions are arrays of regular expressions
+paired with scope names.
+Sublime Text will try to match these patterns
+against a buffer's text
+and attach the corresponding scope name
+to each occurrence.
+These pairs of regular expressions and scope names
+are known as *rules*.
+Sets of rules called *contexts*
+are pushed to and popped from a stack.
+
+### Basic Processing
+
+The basic loop for the syntax engine looks like this:
+
+1. The engine pushes the first context, `main`.
+ It also sets the current character
+ to the first character
+ of the unprocessed editor text.
+
+2. From the current character
+ to the end of its line,
+ evaluate the regular expressions
+ in the current stack frame
+ from first to last
+ until one matches.
+
+3. Consume characters
+ in the matching regexp
+ and assign scopes to them.
+
+4. Optionally push or pop a context on the stack.
+ The `sublime-syntax` format has a significant change
+ from TextMate,
+ wherein multiple stack frames can be pushed or popped at a time.
+ It also supports replacing the current context
+ with a different one.
+
+5. If none of the regular expressions
+ in the current context match
+ at the current position in the file,
+ the engine will advance
+ to the next character of the file
+ and restart the list of regexps
+ in the current context.
+
+6. Go to step 2 at the new character position.
+
+::: warning Caveats
+- Regular expressions that consume no characters
+ should change the stack.
+ If they do not,
+ the current character is advanced
+ as in step 5
+ to avoid an infinite loop.
+ Common examples are
+ lookaheads like `(?=\S)`,
+ BOL or EOL anchors like `^` and `$`,
+ and the null regexp `''` that always matches.
+
+- Regular expressions *do not match* across line breaks.
+
+- Since rules are matched in order,
+ make sure that more specific rules
+ come sooner in each context.
+ Otherwise, a greedy regular expression might swallow parts
+ you'd like to have styled differently.
+:::
+
+
+## Reusing Matches
+
+### `contexts`
+
+It would be a mess to repeat relevant regular expressions
+in each stack frame that needed them.
+This is the purpose of the `contexts` section.
+Contexts are sets of matches and other instructions
+that a stack frame can `include` in its matches.
+They are processed in the same order
+as if they were `match` instructions
+at the location of the `include` instruction.
+
+Contexts can also be pushed onto the stack
+as a new frame.
+This is why stack frames are often
+colloquially referred to as "contexts,"
+including in Sublime Text's scope debugger.
+
+::: tip
+- Contexts can be included from separate files.
+
+- Rules can recurse through pushed contexts.
+ (`main` -> `brace-blocks` -> `main`)
+:::
+
+### `variables`
+
+Portions of regular expressions can also be saved
+in the `variables` section and reused in multiple expressions.
+Variables also make some ugly regexps readable.
+
+
+### Match reuse best practice
+
+Well-designed syntax definitions will define utility contexts
+that `include` equivalent things together for re-usability:
+
+- A normal programming language will have things like
+ - A **statements** group of all things that can be directly executed.
+ This then may or may not (language-dependent) include…
+ - An **expressions** group of things
+ that you can put on the right-hand-side of an assignment,
+ which will definitely include…
+ - An **atoms** group of strings, numbers, chars, etc.
+ that might also be valid statements,
+ but that also depends on your language.
+ - **function-definitions** probably won't be in **expressions**
+ (unless they are lambdas)
+ but probably _would_ be in **statements**.
+ Function definitions might push into a context
+ that lets you `return` and so on.
+
+- A markup language might have
+ - An **inlines** group to keep track of all the markup
+ one can have within a block.
+ - A **blocks** group to hold lists, quotes, paragraphs, headers.
+ - …
+
+
+## Manipulating the Stack
+
+Sublime's syntax definitions allow better stack control
+than TextMate languages,
+allowing some common patterns.
+These are adapted from [the Tips issue][tips]
+on the repository for ST's own syntaxes.
+
+[tips]: https://github.com/sublimehq/Packages/issues/757
+
+
+### Pushing multiple contexts
+
+When you have a construction
+where you expect a list of elements in sequence,
+put them all onto the stack at once.
+The stack will unwind as the elements are recognized.
+
+```yaml
+contexts:
+
+ else-pop:
+ - match: (?=\S)
+ pop: 1
+
+ functions:
+ - match: function(?=\s)
+ scope: keyword.declaration.function
+ push:
+ - function-body
+ - function-params
+ - function-name
+
+ function-name:
+ - match: (?:{{identifier_function}})?(?=[({])
+ scope: entity.name.function
+ pop: 1
+ - include: storage-modifiers # global, private, etc.
+
+ function-params:
+ - match: \(
+ scope: punctuation.section.parameters.begin
+ push: function-param-body
+ - include: else-pop
+
+ function-param-body:
+ - meta_scope: meta.function.parameters
+ - match: \)
+ scope: punctuation.section.parameters.end
+ pop: 2
+ - ...
+
+ function-body:
+ - meta_scope: meta.function
+ - match: \{
+ scope: punctuation.section.block.begin
+ push: function-body-content
+ - include: else-pop
+
+ function-body-content:
+ - match: \}
+ scope: punctuation.section.block.end
+ pop: 2
+ - include: statements
+```
+
+As an added benefit, most of these scopes can be reused:
+
+```yaml
+ immediately-pop:
+ - match: ''
+ pop: 1
+
+ statements:
+ ...
+ - match: \{
+ scope: punctuation.section.braces.begin
+ push:
+ - meta-block
+ - expect-closing-brace
+ - statements
+ ...
+
+ meta-block:
+ - meta_scope: meta.block
+ - include: immediately-pop
+```
+
+As a bonus, states stacked this way are implicitly optional.
+If one is omitted,
+the highlighter will move on to the next without interruption.
+For instance, in the first example,
+the construction will be parsed correctly
+whether or not the author supplies a function name.
+
+::: tip Tip
+Use plural context names to indicate non-popping contexts.
+In other words, plural contexts can match multiple times.
+
+Use singular context names where the contents can only match once.
+:::
+
+
+### Context chaining
+
+You can also manipulate the stack with sequences of `set`s.
+Before making an elaborate state machine,
+ask yourself if you *really* need to.
+
+
+#### Push your first state
+
+While it is absolutely possible to have a match in `main`
+which `set`s into a chain of stateful contexts
+and subsequently sets back into `main` at the end,
+it is not recommended.
+`main` should be a stateless "baseline" context
+that is always the last element on the stack.
+
+Instead, have your match in `main` use push
+to get into your first state,
+then `pop` out of the last state.
+For example, imagine we wanted
+to match the sequence `abc` with each character scoped differently
+and only when they follow each other.
+For illustration purposes, we will also match numerics in `main`:
+
+```yaml
+contexts:
+ main:
+ - match: a
+ scope: first
+ push: expect-b
+ - match: \d+
+ scope: constant.numeric
+
+ expect-b:
+ - match: b
+ scope: second
+ set: expect-c
+
+ expect-c:
+ - match: c
+ scope: third
+ pop: 1
+```
+
+Notice how `a` pushes `expect-b`.
+We don't set the first context, only the second one.
+Once we find the terminator, we pop out.
+
+
+#### Lookahead push for meta scoping
+
+Sometimes you need to apply a meta scope
+to an entire stateful chunk.
+When this is the case,
+you almost certainly want your push rule
+to be a non-consuming lookahead
+rather than a consuming scoped match.
+We can modify the above:
+
+```yaml
+contexts:
+ main:
+ - match: (?=a)
+ push: expect-a
+ - match: \d+
+ scope: constant.numeric
+
+ expect-a:
+ - meta_scope: meta.abc
+ - match: a
+ scope: first
+ set: expect-b
+
+ expect-b:
+ - meta_scope: meta.abc
+ - match: b
+ scope: second
+ set: expect-c
+
+ expect-c:
+ - meta_scope: meta.abc
+ - match: c
+ scope: third
+ pop: 1
+```
+
+
+#### Bail outs
+
+Always remember that you're writing a parser
+for a set of partially valid syntax fragments.
+The normal mode of operation is that someone is actively typing new text.
+For this reason,
+make sure that any and all stateful contexts you use
+have aggressive "bail-outs" for when something goes wrong.
+As a rule of thumb, if there's a case where a compiler's parser would have produced an error,
+your syntax mode should handle that case by `pop`ing back to `main`.
+
+Consider the example from above. Imagine the user is typing typing into the following buffer:
+
+```
+42
+ab
+12
+```
+
+Even if the user is actively typing `c` following `b`,
+it would be a terrible experience for the scoping on `12`
+to shift back and forth as they type in the middle.
+For this reason, you should always end your mid-state scopes
+with a lookahead match like `else-pop` from the multi-push section above
+that pops out of the state chain.
+
+Getting this wrong is one of the easiest ways
+to create a terrible experience for users of your mode
+without even realizing it yourself.
+
+
+## Other Instruction Keywords
+
+Consult [the official documentation][sublime-syntax]
+for more detail on the terms below.
+
+
+### Meta scopes
+
+`meta_scope`
+: Apply a scope to a whole context,
+ including the matches that push and pop it.
+
+`meta_content_scope`
+: Apply a scope to a whole context,
+ except for the matches that push and pop it.
+
+
+### Prototyping
+
+The `prototype` context
+: A special context included
+ at the beginning of every context
+ except contexts included by `prototype` itself.
+ For example: comments
+
+`meta_include_prototype`
+: Keyword to disable including `prototype`.
+ For example: inside strings
+
+`with_prototype`
+: When pushing a context,
+ also include these rules
+ at the beginning of *every* nested context.
+
+
+### Embedding
+
+`embed`
+: Like a pushed context,
+ popping rules are different
+ as described in `escape`.
+
+`escape`
+: Aggressively return directly to the embedding context,
+ popping any number of contexts upon match.
+
+`escape_captures`
+: Allow assigning scopes to the regexp in `escape`.
+
+
+### Branching
+
+Sometimes the appropriate scope is not decidable
+without context beyond a line break.
+For these cases, the `branch` keyword
+describes an array of speculative contexts
+to try until a `fail` match rewinds
+back to the `branch_point`.
+
+
+### Inheritance
+
+Syntaxes can extend from other syntaxes.
+Each variable can be overridden at will.
+Each context can be prepended to, appended to, or replaced outright.
+
+
+## Regular Expression Performance
+
+Sublime built a custom regexp engine to process rules,
+commonly called `sregex`.
+It explicitly excludes support for certain constructs
+that are slow or explode backtracking.
+
+Oniguruma is still available and used where necessary,
+but the best practice for development is to eliminate incompatible patterns.
+
+In practice, this means to avoid
+
+- Anything non-regular in the formal sense
+ (backreferences, recursive matches, etc.).
+ + Except when capture groups are used in a `push`.
+ Those are available in a `pop` as backrefs.
+
+- Lookbehinds
+ + Except in `escape` patterns.
+
+- Atomic groups and possessive quantifiers
+
+- Some Unicode character properties
+
+- Named captures
+
+
+### Testing `sregex` Compatibility
+
+Syntax definitions have a build variant
+to test pattern compatibility with the `sregex` engine.
+Use and choose the
+**Syntax Tests - Regex Compatibility** option.
diff --git a/docs/guide/extensibility/syntax/tutorial.md b/docs/guide/extensibility/syntax/tutorial.md
new file mode 100644
index 00000000..d3761dd9
--- /dev/null
+++ b/docs/guide/extensibility/syntax/tutorial.md
@@ -0,0 +1,586 @@
+---
+title: Syntax Definition Tutorial
+---
+
+# Syntax Definitions
+
+## Prerequisites
+
+In order to follow this tutorial,
+you will need to install [PackageDev][],
+a package intended to ease the creation
+of new syntax definitions for Sublime Text.
+Follow the installation notes
+in the "Getting Started" section of its ReadMe.
+
+[packagedev]: https://github.com/SublimeText/PackageDev
+
+
+## Your First Syntax Definition
+
+By way of example, let's create a syntax definition for Sublime Text
+snippets. We'll be styling the actual snippet content, not the whole
+`.sublime-snippet` file.
+
+::: tip Note
+Since syntax definitions are primarily used to enable syntax highlighting,
+we'll use the phrase *to style* to mean *to break down a source code file
+into scopes*. Keep in mind, however, that colors are a different thing from
+syntax definitions and that scopes have many more uses besides syntax
+highlighting.
+:::
+
+Here are the elements we want to style in a snippet:
+
+- Variables (`$PARAM1`, `$USER_NAME`\ ...)
+- Simple fields (`$0`, `$1`\ ...)
+- Complex fields with placeholders (`${1:Hello}`)
+- Nested fields (`${1:Hello ${2:World}!}`)
+- Escape sequences (`\$`, `\<`, …)
+- Illegal sequences (`$`, `<`, `\`, …)
+
+Here are the elements we don't want to style because they are too complex for
+this example:
+
+- Variable Substitution (`${1/Hello/Hi/g}`)
+
+::: tip Note
+Before continuing, make sure you've installed the
+ package as explained above.
+:::
+
+
+## Creating a New Syntax Definition
+
+To create a new syntax definition, follow these steps:
+
+1. Open the Command Palette and choose **New Syntax…**
+1. Save the new file in your `Packages/User` folder
+ as a `.sublime-syntax` file.
+
+You now should see a file like this:
+
+```yaml
+%YAML 1.2
+---
+# See https://www.sublimetext.com/docs/syntax.html
+file_extensions:
+ - ec
+scope: source.example-c
+contexts:
+ main:
+# ...
+```
+
+Let's examine the key elements.
+
+`scope`
+: The topmost [scope][] for this syntax definition.
+ It takes the form `source.` or `text.`.
+ For programming languages, use `source`.
+ For markup and everything else, use `text`.
+
+`file_extensions`
+: This is a list of file extensions (without the leading dot).
+ When opening files of these types,
+ Sublime Text will automatically activate this syntax definition for them.
+
+`contexts`
+: A container for contexts that contain your match patterns.
+ You'll notice the `main` context
+ from [How Syntax Definitions Work][syndef-work]
+ and some other sample contexts we won't need.
+
+[scope]: index.md#scopes
+[syndef-work]: index.md#how-syntax-definitions-work
+
+For our example, edit the template with the following information,
+and throw away everything after `main:`.
+
+```yaml
+name: Sublime Snippet (Raw)
+scope: source.ssraw
+file_extensions:
+ - ssraw
+
+contexts:
+ main:
+```
+
+::: tip Note
+YAML is not a very strict format, but can cause headaches when you don't
+know its conventions. It supports single and double quotes, but you may also
+omit them as long as the content does not create another YAML literal.
+
+ syntax highlighting is very good
+at demonstrating where strings will be correctly or incorrectly parsed.
+When quotes are necessary, convention is to use single quotes.
+
+The `---` and `...` are optional.
+:::
+
+
+## Creating some rules
+
+If you don't remember [how syntax definition processing works][syndef-work]
+at a general level,
+you may want to read [a synopsis][syndef-work].
+
+Values in the `contexts` dictionary can contain several types of element.
+We'll look at some of them in the following sections.
+If you want to learn more,
+refer to [the official documentation][sublime-syntax].
+
+
+### Matches
+
+Matches take this form:
+
+```yaml
+- match: (?i:m)y \s+[Rr]egex
+ scope: string.format
+```
+
+Sublime Text uses a custom engine called `sregex`
+for regular expressions in syntax definitions
+that uses [Oniguruma][]'s format.
+There is a fallback to the Oniguruma engine
+for features `sregex` doesn't support.
+Several existing syntax definitions make use of
+features supported by Oniguruma that aren't part of
+PERL-style regular expressions, hence the requirement for the fallback.
+
+[oniguruma]: https://github.com/kkos/oniguruma/blob/master/doc/RE
+
+`match`
+: A regular expression Sublime Text will use to find matches.
+
+`scope`
+: The name of the [scope][] that should be applied
+ to any occurrences of `match`.
+
+Let's go back to our example
+and begin to add our rules for Sublime snippets.
+We'll start with simple fields.
+These could be matched with a regex like so:
+
+```perl
+\$[0-9]+
+# or...
+\$\d+
+```
+
+We can then build our pattern like this:
+```yaml
+# Tab stops like $1, $2...
+- match: \$\d+
+ scope: variable.language.ssraw
+```
+
+or even add a capture group to further scope the punctuation:
+```yaml
+# Tab stops like $1, $2...
+- match: (\$)\d+
+ scope: variable.language.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+```
+
+Captures introduce complexity to your rule,
+but they are pretty straightforward.
+Notice how numbers refer to parenthesized groups left to right.
+Of course, you can have as many capture groups as you want.
+
+::: tip Note
+As with usual regular expressions and substitutions, the capture group
+`0` applies to the whole match.
+:::
+
+We choose `variable.language` because the `$1` is a variable,
+but its value is set by the language itself,
+not by an assignment or declaration.
+The `punctuation...` scope lets color schemes carefully target
+the punctuation if their designers want them to.
+
+
+::: tip Choosing the Right Scope Name
+Naming scopes isn't obvious sometimes.
+Check the [naming conventions][] for guidance on scope names.
+ automatically provides completions
+for scope names according to these conventions.
+It is important to re-use the basic categories outlined there
+if you want to achieve the highest compatibility
+with existing [color schemes][].
+
+Color schemes have hardcoded scope names in them. They could not
+possibly include every scope name you can think of, so they target the
+standard ones plus some rarer ones on occasion (like for CSS or
+Markdown). This means that two color schemes using the same syntax
+definition may render the text differently!
+That's left to the color scheme, though.
+Resist any temptation to use novel scopes
+to select a specific color in your color scheme.
+:::
+
+[color schemes]: /guide/customization/color_schemes.md
+[naming conventions]: https://www.sublimetext.com/docs/scope_naming.html
+
+And we can add it to our syntax definition too:
+
+```yaml
+name: Sublime Snippet (Raw)
+scope: source.ssraw
+file_extensions:
+ - ssraw
+
+contexts:
+ main:
+ # Tab stops like $1, $2...
+ - match: (\$)\d+
+ scope: variable.language.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+```
+
+::: tip Note
+You should use two spaces for indent. This is the recommended indent for
+YAML and lines up with lists like shown above.
+:::
+
+You have now created your first syntax definition.
+Next, open a new file and
+save it with the extension `.ssraw`.
+The buffer's syntax name should switch
+to "Sublime Snippet (Raw)" automatically,
+and you should get syntax highlighting
+if you type `$1` or any other simple snippet field.
+
+
+### Defining Variables for Regexes
+
+Let's proceed to creating another rule for environment variables.
+
+```yaml
+# Variables like $SELECTION, $TM_FILEPATH...
+- match: (\$)[A-Za-z][A-Za-z0-9_]*
+ scope: variable.language.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+```
+
+That doesn't quite match what users' experience will be, though:
+not every word is a language-supported placeholder.
+Let's demote the generic case to a generic variable scope
+and add an allowlist of the specific words that snippets recognize.
+Be sure to add the special rule before the catch-all.
+
+```yaml
+# Known variables like $SELECTION, $TM_FILEPATH...
+- match: (\$)(?:SELECTION|TM_SELECTED_TEXT|TM_LINE_INDEX|this gets very long)
+ scope: variable.language.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+
+# Unknown variables or incompletely-typed ones
+- match: (\$)[A-Za-z][A-Za-z0-9_]*
+ scope: variable.other.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+```
+
+That list of variables is actually rather long,
+and we might even want to use it elsewhere in our file.
+Let's pull it out into `variables` and refer to it by name:
+
+```yaml
+contexts:
+ main:
+ # ...
+
+ # Known variables like $SELECTION, $TM_FILEPATH...
+ - match: (\$){{variable_name}}
+ scope: variable.language.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+
+ # Unknown variables or incompletely-typed ones
+ - match: (\$)[A-Za-z][A-Za-z0-9_]*
+ scope: variable.other.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+
+ # ...
+
+variables:
+ variable_name: |-
+ (?x:\b(?:
+ SELECTION
+ | TM_SELECTED_TEXT
+ | TM_LINE_INDEX
+ | TM_LINE_NUMBER
+ | TM_DIRECTORY
+ | TM_FILEPATH
+ | TM_FILENAME
+ | TM_CURRENT_WORD
+ | TM_CURRENT_LINE
+ | TM_TAB_SIZE
+ | TM_SOFT_TABS
+ | TM_SCOPE
+ )\b)
+```
+
+Now `variable_name` holds a regex snippet that can be re-used
+by `match` patterns with `{{variable_name}}`.
+
+
+::: tip Best Practice
+Remember that variables are stamped directly
+into the regex string as parsed by YAML.
+
+Avoid capturing groups if possible.
+They will offset the `captures` in a match.
+
+But wrap your variables in a non-capturing group!
+This lets the match patterns apply quantifiers.
+If you assign variable **boolean** to `true|false`,
+then `{{boolean}}?` will only make the `e` in "false" optional.
+You want `(?:true|false)` instead.
+
+Long variables (or `match` regexes) can use multi-line mode
+for ease of comprehension,
+usually with a YAML block string.
+You can even leave line comments with `#`.
+:::
+
+
+### Push and Pop Rules
+
+Up to now we've been using a simple rule.
+Although we've seen how to dissect patterns into smaller components,
+sometimes you'll want to target a larger portion of your source code
+that is clearly delimited by start and end marks.
+
+From [the synopsis of how syntax engines work][syndef-work],
+you'll remember that the array of rules to match on
+can be changed by manipulating the stack of [contexts](index.md#contexts).
+Literal strings enclosed by quotation marks or other delimiting constructs
+are better dealt with by push and pop rules.
+This is a skeleton for one of these rules
+that pushes an anonymous context,
+namely one that doesn't have an identifier in `contexts`.
+
+```yaml
+- match:
+ scope:
+ push:
+ - match:
+ scope:
+ pop: 1
+```
+
+That is the simplest version.
+Let's take a look at one that includes all available options:
+
+``` yaml
+- match:
+ scope:
+ push:
+ - meta_scope:
+ - meta_content_scope:
+ - match:
+ scope:
+ pop: 1
+# ...
+```
+
+Some elements may look familiar, but their combination might be
+daunting. Let's inspect them individually.
+
+`meta_scope`
+: This sets the following scope name to
+ *the whole context* and the match that pushed it,
+ above any of that match's `scope` or `captures`.
+ **Optional.**
+
+`meta_content_scope`
+: Unlike the `meta_scope`,
+ this only applies a scope name to the portion of the context
+ that does not include the pushing or popping matches.
+ **Optional.**
+
+outer `match`
+: Regex for the opening mark for this scope.
+
+inner `match`
+: Regex for the end mark for this scope.
+
+`pop`
+: Notes a number of contexts to pop off of the stack.
+ Multiple matches can have `pop` instructions.
+ **Optional.**
+
+We'll use this rule to style nested complex fields in snippets:
+
+```yaml
+- match: (\$)(\{)[0-9]+(:)
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+ 2: punctuation.section.interpolation.begin.ssraw
+ 3: punctuation.separator.ssraw
+ push:
+ - meta_scope: variable.language.complex.ssraw
+ - meta_content_scope: string.unquoted.ssraw
+ - match: \}
+ scope: punctuation.section.interpolation.end.ssraw
+ pop: 1
+ - include: main
+```
+
+Although it is possible to push anonymous contexts,
+best practice is to name them for ease of debugging.
+
+Let's give this context a name:
+
+```yaml
+contexts:
+ main:
+ # ... (other rules) ...
+
+ # Complex variables ${: ... }
+ - match: (\$)(\{)[0-9]+(:)
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+ 2: punctuation.section.interpolation.begin.ssraw
+ 3: punctuation.separator.ssraw
+ push: complex_variable_body
+
+ complex_variable_body:
+ - meta_scope: variable.language.complex.ssraw
+ - meta_content_scope: string.unquoted.ssraw
+ - match: \}
+ scope: punctuation.section.interpolation.end.ssraw
+ pop: 1
+ - include: main
+```
+
+This is the most complex pattern we'll see in this tutorial.
+
+Notice that there are other matches and includes
+that do not pop the context stack.
+These will be matched until a `}` is encountered.
+It even includes **the `main` context,**
+which happily recurses if another `${\d` match is discovered!
+
+Remember, [matched text is consumed][syndef-work].
+It is consequently excluded from the next match attempt
+and can't be matched again.
+Make sure your additional matches **do not**
+accidentally eat the popping match.
+
+To finish off complex fields,
+we've styled the placeholders as strings
+with the `meta_content_scope` field.
+
+
+### Final Touches
+
+Lastly, let's style escape sequences and illegal sequences, and then we
+can wrap up.
+
+```yaml
+# Sequences like \$, \> and \<
+- match: \\[$<>]
+ scope: constant.character.escape.ssraw
+
+# Unescaped and unmatched magic characters
+- match: '[$<>]'
+ scope: invalid.illegal.ssraw
+```
+
+The only hard thing here is not forgetting that `[]` enclose arrays in
+YAML and thus must be wrapped in quotes. Other than that, the rules are
+pretty straightforward if you're familiar with regular expressions.
+
+However, you must take care to place the second rule after any others
+matching the `$` character, since otherwise it will be consumed and
+result in every following expression not matching.
+
+Also, even after adding these two additional rules, note that our
+recursive begin-end rule from above continues to work as expected.
+
+At long last, here's the final syntax definition:
+
+```yaml
+%YAML 1.2
+---
+# See https://www.sublimetext.com/docs/syntax.html
+name: Sublime Snippet (Raw)
+scope: source.ssraw
+file_extensions:
+ - ssraw
+
+contexts:
+ main:
+ # Tab stops like $1, $2...
+ - match: (\$)\d+
+ scope: variable.language.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+
+ # Known variables like $SELECTION, $TM_FILEPATH...
+ - match: (\$){{variable_name}}
+ scope: variable.language.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+
+ # Unknown variables or incompletely-typed ones
+ - match: (\$)[A-Za-z][A-Za-z0-9_]*
+ scope: variable.other.ssraw
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+
+ # Complex variables ${: ... }
+ - match: (\$)(\{)[0-9]+(:)
+ captures:
+ 1: punctuation.definition.variable.begin.ssraw
+ 2: punctuation.section.interpolation.begin.ssraw
+ 3: punctuation.separator.ssraw
+ push: complex_variable_body
+
+ # Sequences like \$, \> and \<
+ - match: \\[$<>]
+ scope: constant.character.escape.ssraw
+
+ # Unescaped and unmatched magic characters
+ - match: '[$<>]'
+ scope: invalid.illegal.ssraw
+
+ complex_variable_body:
+ - meta_scope: variable.language.complex.ssraw
+ - meta_content_scope: string.unquoted.ssraw
+ - match: \}
+ scope: punctuation.section.interpolation.end.ssraw
+ pop: 1
+ - include: main
+
+variables:
+ variable_name: |-
+ (?x:\b(?:
+ SELECTION
+ | TM_SELECTED_TEXT
+ | TM_LINE_INDEX
+ | TM_LINE_NUMBER
+ | TM_DIRECTORY
+ | TM_FILEPATH
+ | TM_FILENAME
+ | TM_CURRENT_WORD
+ | TM_CURRENT_LINE
+ | TM_TAB_SIZE
+ | TM_SOFT_TABS
+ | TM_SCOPE
+ )\b)
+```
+
+There are more available constructs and code reuse techniques,
+but the above explanations should get you started with the
+creation of syntax definitions.
diff --git a/docs/guide/extensibility/syntaxdefs.md b/docs/guide/extensibility/syntax/tutorial_legacy.md
similarity index 86%
rename from docs/guide/extensibility/syntaxdefs.md
rename to docs/guide/extensibility/syntax/tutorial_legacy.md
index 8b2321fe..baec0571 100644
--- a/docs/guide/extensibility/syntaxdefs.md
+++ b/docs/guide/extensibility/syntax/tutorial_legacy.md
@@ -1,8 +1,8 @@
---
-title: Syntax Definitions
+title: Syntax Definition Tutorial (Legacy)
---
-# Syntax Definitions
+# Syntax Definition Tutorial (Legacy)
Syntax definitions make Sublime Text aware of programming and markup languages.
Most noticeably, they work together with colors to provide syntax highlighting.
@@ -65,65 +65,6 @@ XML, but always keep in mind their differing needs in regards to escape
sequences, many XML tags etc.
-## Scopes
-
-Scopes are a key concept in Sublime Text. Essentially, they are named
-text regions in a buffer. They don't do anything by themselves, but
-Sublime Text peeks at them when it needs contextual information.
-
-For instance, when you trigger a snippet, Sublime Text checks the scope
-bound to the snippet and looks at the caret's position in the file. If
-the caret's current position matches the snippet's scope selector,
-Sublime Text fires it off. Otherwise, nothing happens.
-
-::: tip Info
-There's a slight difference between *scopes* and *scope selectors*: Scopes
-are the names defined in a syntax definition, while scope selectors are used
-in items like snippets and key bindings to target scopes. When creating a
-new syntax definition, you care about scopes; when you want to constrain a
-snippet to a certain scope, you use a scope selector.
-:::
-
-Scopes can be nested to allow for a high degree of granularity. You can drill
-down the hierarchy very much like with CSS selectors. For instance, thanks to
-scope selectors, you could have a key binding activated only within single
-quoted strings in Python source code, but not inside single quoted strings in
-any other language.
-
-Sublime Text inherits the idea of scopes from Textmate, a text editor for Mac.
-[Textmate's online manual][] contains further information about scope selectors
-that's useful for Sublime Text users too. In particular, Color Schemes make
-extensive use of scopes to style every aspect of a language in the desired
-color.
-
-[Textmate's online manual]: https://manual.macromates.com/en/scope_selectors
-
-
-## How Syntax Definitions Work
-
-At their core, syntax definitions are arrays of regular expressions
-paired with scope names. Sublime Text will try to match these patterns
-against a buffer's text and attach the corresponding scope name to all
-occurrences. These pairs of regular expressions and scope names are
-known as *rules*.
-
-Rules are applied in order, one line at a time. Rules are applied in the
-following order:
-
-1. The rule that matches at the first position in a line
-2. The rule that comes first in the array
-
-Each rule consumes the matched text region, which therefore will be
-excluded from the next rule's matching attempt (save for a few
-exceptions). In practical terms, this means that you should take care to
-go from more specific rules to more general ones when you create a new
-syntax definition. Otherwise, a greedy regular expression might swallow
-parts you'd like to have styled differently.
-
-Syntax definitions from separate files can be combined, and they can be
-recursively applied too.
-
-
## Your First Syntax Definition
By way of example, let's create a syntax definition for Sublime Text
@@ -407,7 +348,7 @@ Arguably, you'd want the other scope to be visually consistent with this one.
Go ahead and change it too.
::: tip Note
-As with ususal regular expressions and substitutions, the capture group
+As with usual regular expressions and substitutions, the capture group
`'0'` applies to the whole match.
:::
diff --git a/docs/reference/completions.md b/docs/reference/completions.md
index 9e3ac752..293ac2d6 100644
--- a/docs/reference/completions.md
+++ b/docs/reference/completions.md
@@ -52,7 +52,7 @@ Here's an example (with HTML completions):
See [Scopes][] for more information.
-[Scopes]: /guide/extensibility/syntaxdefs.md#scopes
+[Scopes]: /guide/extensibility/syntax/index.md#scopes
**completions**
: Array of *completions*.