feat: tumblr grammar with highlights, injections, and tests

2026-05-17 05:50:17 +02:00
parent 1596154284
commit c61fa5c53b
11 changed files with 1950 additions and 55 deletions
@@ -10,8 +10,64 @@
 export default grammar({
  name: "tumblr",

+  extras: _ => [],
+
  rules: {
-    // TODO: add the actual grammar rules
-    source_file: $ => "hello"
-  }
+    template: $ => repeat($._node),
+
+    _node: $ => choice(
+      $.content,
+      $.block_open,
+      $.block_close,
+      $.lang_tag,
+      $.variable,
+    ),
+
+    content: _ => token(prec(-1, /([^{]|\{[^A-Za-z/])+/)),
+
+    block_open: $ => seq(
+      $._block_open_start,
+      $.block_name,
+      optional($.attributes),
+      "}",
+    ),
+
+    block_close: $ => seq(
+      $._block_close_start,
+      $.block_name,
+      "}",
+    ),
+
+    lang_tag: $ => seq(
+      $._lang_start,
+      $.lang_text,
+      "}",
+    ),
+
+    variable: $ => seq(
+      "{",
+      choice(
+        seq($.variable_name, optional(seq("-", $.variable_modifier))),
+        seq($.variable_prefix, ":", $.prefix_argument),
+      ),
+      "}",
+    ),
+
+    attributes: $ => repeat1($.attribute),
+    attribute: $ => seq($._space, $.attribute_name, "=", $.attribute_value),
+    attribute_name: _ => /[A-Za-z_][A-Za-z0-9_-]*/,
+    attribute_value: _ => /"[^"]*"/,
+
+    block_name: _ => /[A-Za-z][A-Za-z0-9_]*/,
+    variable_name: _ => /[A-Z][A-Za-z0-9_]*/,
+    variable_modifier: _ => /[A-Za-z0-9]+/,
+    variable_prefix: _ => choice("text", "color", "font", "image"),
+    prefix_argument: _ => /[A-Za-z][A-Za-z0-9 _-]*/,
+    lang_text: _ => /[^}]+/,
+
+    _block_open_start: _ => /\{[Bb][Ll][Oo][Cc][Kk]:/,
+    _block_close_start: _ => /\{\/[Bb][Ll][Oo][Cc][Kk]:/,
+    _lang_start: _ => /\{[Ll][Aa][Nn][Gg]:/,
+    _space: _ => /[ \t]+/,
+  },
 });
@@ -0,0 +1,101 @@
+; Punctuation
+"{" @punctuation.bracket
+"}" @punctuation.bracket
+
+; Keywords that introduce a tag form. The grammar exposes the opening
+; sequence as a hidden node, so capture the parent and let the editor
+; colour the leading {block:, {/block:, {lang: literally via the
+; tokenizer. The colon delimiter inside a variable_prefix tag is captured
+; below.
+":" @punctuation.delimiter
+"-" @punctuation.delimiter
+"=" @operator
+
+; Block names. Known data-block names get @function.builtin; If/IfNot
+; toggles get @keyword.conditional (theme authors may define arbitrary
+; If* / IfNot* names via <meta name="if:..."> so we match by prefix).
+((block_name) @keyword.conditional
+ (#match? @keyword.conditional "^[Ii]f([Nn]ot)?[A-Z]"))
+
+((block_name) @function.builtin
+ (#any-of? @function.builtin
+  "Album" "AlbumArt" "Answer" "Answerer" "Aperture" "Artist" "AskEnabled"
+  "Audio" "AudioEmbed" "AudioPlayer" "Author" "Camera" "Caption" "Chat"
+  "ContentSource" "CurrentPage" "Date" "DayPage" "DayPagination"
+  "Description" "Even" "Excerpt" "Exif" "Exposure" "ExternalAudio"
+  "FeaturedTags" "FocalLength" "Followed" "Following" "GroupMember"
+  "GroupMembers" "HasAvatar" "HasFeaturedTags" "HasPages" "HasPermalink"
+  "HasTags" "HideAvatar" "HideDescription" "HideFromSearchEnabled"
+  "HideHeaderImage" "HideTitle" "HighRes" "HomePage" "Host" "IndexPage"
+  "IsActive" "IsDeactivated" "isOriginalEntry" "JumpPage" "Label"
+  "LikeCount" "Likes" "Lines" "Link" "LinkURL" "More" "NewDayDate"
+  "NextDayPage" "NextPage" "NextPost" "NoLikes" "NoSearchResults"
+  "NoSourceLogo" "NoteCount" "NotReblog" "Odd" "Pages" "Pagination"
+  "Panorama" "PermalinkPage" "PermalinkPagination" "Photo" "Photos"
+  "Photoset" "PinnedPostLabel" "PlayCount" "Post5" "PostNotes" "Posts"
+  "PostSummary" "PostTitle" "PreviousDayPage" "PreviousPage"
+  "PreviousPost" "Quote" "ReblogCount" "RebloggedFrom" "Reblogs"
+  "RelatedPosts" "ReplyCount" "SameDayDate" "SearchPage" "ShowAvatar"
+  "ShowDescription" "ShowHeaderImage" "ShowTitle" "Source" "SourceLogo"
+  "Submission" "SubmissionsEnabled" "TagPage" "Tags" "Text" "Thumbnail"
+  "Title" "TrackName" "Video" "VideoThumbnail" "VideoThumbnails"))
+
+(block_name) @function
+
+; Variable names. Known builtins get @variable.builtin, others get @variable.
+((variable_name) @variable.builtin
+ (#any-of? @variable.builtin
+  "AccentColor" "Album" "AlbumArtURL" "Alt" "AmPm" "Answer" "Answerer"
+  "AnswererPortraitURL" "Aperture" "Artist" "Asker" "AskerPortraitURL"
+  "AskLabel" "AudioEmbed" "AudioPlayer" "Author" "AvatarShape"
+  "BackgroundColor" "Beats" "BlackLogoURL" "BlogURL" "Body" "Camera"
+  "CapitalAmPm" "Caption" "CopyrightYears" "CurrentPage" "CustomCSS"
+  "DayOfMonth" "DayOfMonthSuffix" "DayOfMonthWithZero" "DayOfWeek"
+  "DayOfWeekNumber" "DayOfYear" "Description" "EmbedUrl" "Excerpt"
+  "Exposure" "ExternalAudioURL" "Favicon" "FocalLength" "FollowedName"
+  "FollowedPortraitURL" "FollowedTitle" "FollowedURL" "FormattedPlayCount"
+  "GroupMemberName" "GroupMemberPortraitURL" "GroupMemberTitle"
+  "GroupMemberURL" "HeaderImage" "Host" "JSDescription" "JSPhotosetLayout"
+  "JSPlaintextDescription" "Label" "Length" "LikeButton" "LikeCount"
+  "Likes" "Line" "LinkCloseTag" "LinkOpenTag" "LinkURL" "LogoHeight"
+  "LogoWidth" "MetaDescription" "Minutes" "Month" "MonthNumber"
+  "MonthNumberWithZero" "Name" "NextDayPage" "NextPage" "NextPost"
+  "NoteCount" "NoteCountWithLabel" "NPF" "PageNumber" "Permalink"
+  "PhotoAlt" "PhotoCount" "PhotoHeight" "Photoset" "PhotosetLayout"
+  "PhotoURL" "PhotoWidth" "PinnedPostLabel" "PlaintextName" "PlayCount"
+  "PlayCountWithLabel" "PortraitURL" "PostAuthorName"
+  "PostAuthorPortraitURL" "PostAuthorTitle" "PostAuthorURL" "PostID"
+  "PostNotes" "PostNotesURL" "PostSummary" "PostTitle" "PostType"
+  "PreviousDayPage" "PreviousPage" "PreviousPost" "Question" "Quote"
+  "RawAudioURL" "ReblogButton" "ReblogCount" "ReblogParentName"
+  "ReblogParentPortraitURL" "ReblogParentTitle" "ReblogParentURL"
+  "ReblogRootName" "ReblogRootPortraitURL" "ReblogRootTitle"
+  "ReblogRootURL" "RelativePermalink" "Replies" "ReplyCount" "RSS"
+  "SearchQuery" "SearchResultCount" "Seconds" "ShortDayOfWeek"
+  "ShortMonth" "ShortURL" "ShortYear" "Source" "SourceTitle" "SourceURL"
+  "SubmitLabel" "Submitter" "SubmitterPortraitURL" "SubmitterURL" "Tag"
+  "TagsAsClasses" "TagURL" "TagURLChrono" "Target" "Thumbnail" "TimeAgo"
+  "Timestamp" "Title" "TitleColor" "TitleFont" "TitleFontWeight"
+  "TotalPages" "TrackName" "URL" "URLEncodedPermalink"
+  "URLSafeSearchQuery" "URLSafeTag" "Username" "UserNumber" "Video"
+  "VideoEmbed" "VideoThumbnailURL" "WeekOfYear" "Year"))
+
+(variable_name) @variable
+
+; Size suffix on URL-style variables. Numeric forms read as numbers,
+; named forms (HighRes, Panorama, ...) as constants.
+((variable_modifier) @number
+ (#match? @number "^[0-9]+(sq)?$"))
+(variable_modifier) @constant.builtin
+
+; text: / color: / font: / image: prefix and its argument.
+(variable_prefix) @keyword
+(prefix_argument) @variable
+
+; {lang:Translatable string}
+(lang_tag) @string.special
+(lang_text) @string
+
+; Block attributes: {block:Photoset rows="3"}
+(attribute_name) @attribute
+(attribute_value) @string
@@ -0,0 +1,7 @@
+; All non-tag content is HTML. injection.combined concatenates every
+; content node so an HTML tag opened in one chunk can close in another
+; (across a {block:Foo}...{/block:Foo}). HTML's own injection queries
+; then take over for <style> and <script>, giving us CSS and JS for free.
+((content) @injection.content
+ (#set! injection.language "html")
+ (#set! injection.combined))
@@ -2,17 +2,272 @@
  "$schema": "https://tree-sitter.github.io/tree-sitter/assets/schemas/grammar.schema.json",
  "name": "tumblr",
  "rules": {
-    "source_file": {
-      "type": "STRING",
-      "value": "hello"
+    "template": {
+      "type": "REPEAT",
+      "content": {
+        "type": "SYMBOL",
+        "name": "_node"
      }
    },
-  "extras": [
+    "_node": {
+      "type": "CHOICE",
+      "members": [
        {
-      "type": "PATTERN",
-      "value": "\\s"
+          "type": "SYMBOL",
+          "name": "content"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "block_open"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "block_close"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "lang_tag"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "variable"
        }
-  ],
+      ]
+    },
+    "content": {
+      "type": "TOKEN",
+      "content": {
+        "type": "PREC",
+        "value": -1,
+        "content": {
+          "type": "PATTERN",
+          "value": "([^{]|\\{[^A-Za-z/])+"
+        }
+      }
+    },
+    "block_open": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "_block_open_start"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "block_name"
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "attributes"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
+        {
+          "type": "STRING",
+          "value": "}"
+        }
+      ]
+    },
+    "block_close": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "_block_close_start"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "block_name"
+        },
+        {
+          "type": "STRING",
+          "value": "}"
+        }
+      ]
+    },
+    "lang_tag": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "_lang_start"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "lang_text"
+        },
+        {
+          "type": "STRING",
+          "value": "}"
+        }
+      ]
+    },
+    "variable": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "{"
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "variable_name"
+                },
+                {
+                  "type": "CHOICE",
+                  "members": [
+                    {
+                      "type": "SEQ",
+                      "members": [
+                        {
+                          "type": "STRING",
+                          "value": "-"
+                        },
+                        {
+                          "type": "SYMBOL",
+                          "name": "variable_modifier"
+                        }
+                      ]
+                    },
+                    {
+                      "type": "BLANK"
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              "type": "SEQ",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "variable_prefix"
+                },
+                {
+                  "type": "STRING",
+                  "value": ":"
+                },
+                {
+                  "type": "SYMBOL",
+                  "name": "prefix_argument"
+                }
+              ]
+            }
+          ]
+        },
+        {
+          "type": "STRING",
+          "value": "}"
+        }
+      ]
+    },
+    "attributes": {
+      "type": "REPEAT1",
+      "content": {
+        "type": "SYMBOL",
+        "name": "attribute"
+      }
+    },
+    "attribute": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "_space"
+        },
+        {
+          "type": "SYMBOL",
+          "name": "attribute_name"
+        },
+        {
+          "type": "STRING",
+          "value": "="
+        },
+        {
+          "type": "SYMBOL",
+          "name": "attribute_value"
+        }
+      ]
+    },
+    "attribute_name": {
+      "type": "PATTERN",
+      "value": "[A-Za-z_][A-Za-z0-9_-]*"
+    },
+    "attribute_value": {
+      "type": "PATTERN",
+      "value": "\"[^\"]*\""
+    },
+    "block_name": {
+      "type": "PATTERN",
+      "value": "[A-Za-z][A-Za-z0-9_]*"
+    },
+    "variable_name": {
+      "type": "PATTERN",
+      "value": "[A-Z][A-Za-z0-9_]*"
+    },
+    "variable_modifier": {
+      "type": "PATTERN",
+      "value": "[A-Za-z0-9]+"
+    },
+    "variable_prefix": {
+      "type": "CHOICE",
+      "members": [
+        {
+          "type": "STRING",
+          "value": "text"
+        },
+        {
+          "type": "STRING",
+          "value": "color"
+        },
+        {
+          "type": "STRING",
+          "value": "font"
+        },
+        {
+          "type": "STRING",
+          "value": "image"
+        }
+      ]
+    },
+    "prefix_argument": {
+      "type": "PATTERN",
+      "value": "[A-Za-z][A-Za-z0-9 _-]*"
+    },
+    "lang_text": {
+      "type": "PATTERN",
+      "value": "[^}]+"
+    },
+    "_block_open_start": {
+      "type": "PATTERN",
+      "value": "\\{[Bb][Ll][Oo][Cc][Kk]:"
+    },
+    "_block_close_start": {
+      "type": "PATTERN",
+      "value": "\\{\\/[Bb][Ll][Oo][Cc][Kk]:"
+    },
+    "_lang_start": {
+      "type": "PATTERN",
+      "value": "\\{[Ll][Aa][Nn][Gg]:"
+    },
+    "_space": {
+      "type": "PATTERN",
+      "value": "[ \\t]+"
+    }
+  },
+  "extras": [],
  "conflicts": [],
  "precedences": [],
  "externals": [],
@@ -1,12 +1,217 @@
 [
  {
-    "type": "source_file",
+    "type": "attribute",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "attribute_name",
+          "named": true
+        },
+        {
+          "type": "attribute_value",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "attributes",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "attribute",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "block_close",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "block_name",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "block_open",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "attributes",
+          "named": true
+        },
+        {
+          "type": "block_name",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "lang_tag",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "lang_text",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "template",
    "named": true,
    "root": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "block_close",
+          "named": true
+        },
+        {
+          "type": "block_open",
+          "named": true
+        },
+        {
+          "type": "content",
+          "named": true
+        },
+        {
+          "type": "lang_tag",
+          "named": true
+        },
+        {
+          "type": "variable",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "variable",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": true,
+      "types": [
+        {
+          "type": "prefix_argument",
+          "named": true
+        },
+        {
+          "type": "variable_modifier",
+          "named": true
+        },
+        {
+          "type": "variable_name",
+          "named": true
+        },
+        {
+          "type": "variable_prefix",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "variable_prefix",
+    "named": true,
    "fields": {}
  },
  {
-    "type": "hello",
+    "type": "-",
+    "named": false
+  },
+  {
+    "type": ":",
+    "named": false
+  },
+  {
+    "type": "=",
+    "named": false
+  },
+  {
+    "type": "attribute_name",
+    "named": true
+  },
+  {
+    "type": "attribute_value",
+    "named": true
+  },
+  {
+    "type": "block_name",
+    "named": true
+  },
+  {
+    "type": "color",
+    "named": false
+  },
+  {
+    "type": "content",
+    "named": true
+  },
+  {
+    "type": "font",
+    "named": false
+  },
+  {
+    "type": "image",
+    "named": false
+  },
+  {
+    "type": "lang_text",
+    "named": true
+  },
+  {
+    "type": "prefix_argument",
+    "named": true
+  },
+  {
+    "type": "text",
+    "named": false
+  },
+  {
+    "type": "variable_modifier",
+    "named": true
+  },
+  {
+    "type": "variable_name",
+    "named": true
+  },
+  {
+    "type": "{",
+    "named": false
+  },
+  {
+    "type": "}",
    "named": false
  }
 ]
@@ -0,0 +1,139 @@
+==================
+Simple block
+==================
+
+{block:Posts}{/block:Posts}
+
+---
+
+(template
+  (content)
+  (block_open
+    (block_name))
+  (block_close
+    (block_name))
+  (content))
+
+==================
+Block with content
+==================
+
+{block:Posts}<article>{Body}</article>{/block:Posts}
+
+---
+
+(template
+  (content)
+  (block_open
+    (block_name))
+  (content)
+  (variable
+    (variable_name))
+  (content)
+  (block_close
+    (block_name))
+  (content))
+
+==================
+Nested blocks
+==================
+
+{block:Posts}{block:Text}{Body}{/block:Text}{/block:Posts}
+
+---
+
+(template
+  (content)
+  (block_open
+    (block_name))
+  (block_open
+    (block_name))
+  (variable
+    (variable_name))
+  (block_close
+    (block_name))
+  (block_close
+    (block_name))
+  (content))
+
+==================
+Block with one attribute
+==================
+
+{block:Photoset rows="3"}{Photoset}{/block:Photoset}
+
+---
+
+(template
+  (content)
+  (block_open
+    (block_name)
+    (attributes
+      (attribute
+        (attribute_name)
+        (attribute_value))))
+  (variable
+    (variable_name))
+  (block_close
+    (block_name))
+  (content))
+
+==================
+Block with multiple attributes
+==================
+
+{block:Photoset rows="3" gutter="10px"}{/block:Photoset}
+
+---
+
+(template
+  (content)
+  (block_open
+    (block_name)
+    (attributes
+      (attribute
+        (attribute_name)
+        (attribute_value))
+      (attribute
+        (attribute_name)
+        (attribute_value))))
+  (block_close
+    (block_name))
+  (content))
+
+==================
+Case insensitive block keyword
+==================
+
+{Block:Photo}{/Block:Photo}
+
+---
+
+(template
+  (content)
+  (block_open
+    (block_name))
+  (block_close
+    (block_name))
+  (content))
+
+==================
+Conditional block
+==================
+
+{block:IfShowSidebar}{block:IfNotHidden}content{/block:IfNotHidden}{/block:IfShowSidebar}
+
+---
+
+(template
+  (content)
+  (block_open
+    (block_name))
+  (block_open
+    (block_name))
+  (content)
+  (block_close
+    (block_name))
+  (block_close
+    (block_name))
+  (content))
@@ -0,0 +1,38 @@
+==================
+Plain HTML, no tags
+==================
+
+<html><body><h1>Hello</h1></body></html>
+
+---
+
+(template
+  (content))
+
+==================
+Stray opening brace
+==================
+
+function foo() { return 1; }
+
+---
+
+(template
+  (content))
+
+==================
+CSS embedded in style
+==================
+
+<style>
+  body { background: {color:Background}; }
+</style>
+
+---
+
+(template
+  (content)
+  (variable
+    (variable_prefix)
+    (prefix_argument))
+  (content))
@@ -0,0 +1,27 @@
+==================
+Simple lang tag
+==================
+
+{lang:Source}
+
+---
+
+(template
+  (content)
+  (lang_tag
+    (lang_text))
+  (content))
+
+==================
+Lang tag with spaces and punctuation
+==================
+
+{lang:Read more, please}
+
+---
+
+(template
+  (content)
+  (lang_tag
+    (lang_text))
+  (content))
@@ -0,0 +1,103 @@
+==================
+Plain variable
+==================
+
+{Title}
+
+---
+
+(template
+  (content)
+  (variable
+    (variable_name))
+  (content))
+
+==================
+Variable with numeric size suffix
+==================
+
+{PortraitURL-128}
+
+---
+
+(template
+  (content)
+  (variable
+    (variable_name)
+    (variable_modifier))
+  (content))
+
+==================
+Variable with named size suffix
+==================
+
+{PhotoURL-HighRes}
+
+---
+
+(template
+  (content)
+  (variable
+    (variable_name)
+    (variable_modifier))
+  (content))
+
+==================
+Multiple variables with text between
+==================
+
+<h1>{Title}</h1>
+<p>{Description}</p>
+
+---
+
+(template
+  (content)
+  (variable
+    (variable_name))
+  (content)
+  (variable
+    (variable_name))
+  (content))
+
+==================
+Prefixed variable
+==================
+
+{color:Background} {font:Body} {image:Header} {text:Tagline}
+
+---
+
+(template
+  (content)
+  (variable
+    (variable_prefix)
+    (prefix_argument))
+  (content)
+  (variable
+    (variable_prefix)
+    (prefix_argument))
+  (content)
+  (variable
+    (variable_prefix)
+    (prefix_argument))
+  (content)
+  (variable
+    (variable_prefix)
+    (prefix_argument))
+  (content))
+
+==================
+Prefix argument with spaces
+==================
+
+{text:Header Subtitle}
+
+---
+
+(template
+  (content)
+  (variable
+    (variable_prefix)
+    (prefix_argument))
+  (content))
@@ -20,13 +20,11 @@
    "authors": [
      {
        "name": "Oscar Wallberg",
-        "email": "oscar.wallberg@outlook.com",
-        "url": ""
+        "email": "oscar.wallberg@outlook.com"
      }
    ],
    "links": {
-      "repository": "git.owall.dev/warg/tree-sitter-tumblr",
-      "funding": ""
+      "repository": "https://git.owall.dev/warg/tree-sitter-tumblr"
    }
  },
  "bindings": {