Rework language boundary detection (#502)

* Fix `classRegex` error

* Rework language boundary detection

packages/tailwindcss-language-service/package.json

@@ -29,6 +29,7 @@
     "semver": "7.3.2",
     "sift-string": "0.0.2",
     "stringify-object": "3.3.0",
+    "tmp-cache": "1.1.0",
     "vscode-emmet-helper-bundled": "0.0.1",
     "vscode-languageclient": "7.0.0",
     "vscode-languageserver": "7.0.0",

packages/tailwindcss-language-service/src/codeActions/provideInvalidApplyCodeActions.ts

@@ -43,7 +43,9 @@ export async function provideInvalidApplyCodeActions(
   if (!isCssDoc(state, document)) {
     let languageBoundaries = getLanguageBoundaries(state, document)
     if (!languageBoundaries) return []
-    cssRange = languageBoundaries.css.find((range) => isWithinRange(diagnostic.range.start, range))
+    cssRange = languageBoundaries
+      .filter((b) => b.type === 'css')
+      .find(({ range }) => isWithinRange(diagnostic.range.start, range))?.range
     if (!cssRange) return []
     cssText = document.getText(cssRange)
   }

packages/tailwindcss-language-service/src/completionProvider.ts

@@ -20,7 +20,7 @@ import { stringifyScreen, Screen } from './util/screens'
 import isObject from './util/isObject'
 import * as emmetHelper from 'vscode-emmet-helper-bundled'
 import { isValidLocationForEmmetAbbreviation } from './util/isValidLocationForEmmetAbbreviation'
-import { isJsContext } from './util/js'
+import { isJsDoc, isJsxContext } from './util/js'
 import { naturalExpand } from './util/naturalExpand'
 import semver from 'semver'
 import { docsUrl } from './util/docsUrl'
@@ -511,7 +511,7 @@ async function provideClassNameCompletions(
     return provideAtApplyCompletions(state, document, position)
   }
 
-  if (isHtmlContext(state, document, position) || isJsContext(state, document, position)) {
+  if (isHtmlContext(state, document, position) || isJsxContext(state, document, position)) {
     return provideClassAttributeCompletions(state, document, position, context)
   }
 
@@ -973,8 +973,8 @@ async function provideEmmetCompletions(
   let settings = await state.editor.getConfiguration(document.uri)
   if (settings.tailwindCSS.emmetCompletions !== true) return null
 
-  const isHtml = isHtmlContext(state, document, position)
-  const isJs = !isHtml && isJsContext(state, document, position)
+  const isHtml = !isJsDoc(state, document) && isHtmlContext(state, document, position)
+  const isJs = isJsDoc(state, document) || isJsxContext(state, document, position)
 
   const syntax = isHtml ? 'html' : isJs ? 'jsx' : null
 

packages/tailwindcss-language-service/src/diagnostics/getInvalidConfigPathDiagnostics.ts

@@ -173,7 +173,7 @@ export function getInvalidConfigPathDiagnostics(
   } else {
     let boundaries = getLanguageBoundaries(state, document)
     if (!boundaries) return []
-    ranges.push(...boundaries.css)
+    ranges.push(...boundaries.filter((b) => b.type === 'css').map(({ range }) => range))
   }
 
   ranges.forEach((range) => {

packages/tailwindcss-language-service/src/diagnostics/getInvalidScreenDiagnostics.ts

@@ -24,7 +24,7 @@ export function getInvalidScreenDiagnostics(
   } else {
     let boundaries = getLanguageBoundaries(state, document)
     if (!boundaries) return []
-    ranges.push(...boundaries.css)
+    ranges.push(...boundaries.filter((b) => b.type === 'css').map(({ range }) => range))
   }
 
   ranges.forEach((range) => {

packages/tailwindcss-language-service/src/diagnostics/getInvalidTailwindDirectiveDiagnostics.ts

@@ -24,7 +24,7 @@ export function getInvalidTailwindDirectiveDiagnostics(
   } else {
     let boundaries = getLanguageBoundaries(state, document)
     if (!boundaries) return []
-    ranges.push(...boundaries.css)
+    ranges.push(...boundaries.filter((b) => b.type === 'css').map(({ range }) => range))
   }
 
   let notSemicolonLanguages = ['sass', 'sugarss', 'stylus']

packages/tailwindcss-language-service/src/diagnostics/getInvalidVariantDiagnostics.ts

@@ -28,7 +28,7 @@ export function getInvalidVariantDiagnostics(
   } else {
     let boundaries = getLanguageBoundaries(state, document)
     if (!boundaries) return []
-    ranges.push(...boundaries.css)
+    ranges.push(...boundaries.filter((b) => b.type === 'css').map(({ range }) => range))
   }
 
   let possibleVariants = Object.keys(state.variants)

packages/tailwindcss-language-service/src/util/css.ts

@@ -1,8 +1,9 @@
 import type { TextDocument, Position } from 'vscode-languageserver'
-import { isInsideTag, isVueDoc, isSvelteDoc, isHtmlDoc } from './html'
+import { isVueDoc, isSvelteDoc, isHtmlDoc } from './html'
 import { isJsDoc } from './js'
 import { State } from './state'
 import { cssLanguages } from './languages'
+import { getLanguageBoundaries } from './getLanguageBoundaries'
 
 export function isCssDoc(state: State, doc: TextDocument): boolean {
   const userCssLanguages = Object.keys(state.editor.userLanguages).filter((lang) =>
@@ -23,7 +24,9 @@ export function isCssContext(state: State, doc: TextDocument, position: Position
       end: position,
     })
 
-    return isInsideTag(str, ['style'])
+    let boundaries = getLanguageBoundaries(state, doc, str)
+
+    return boundaries ? boundaries[boundaries.length - 1].type === 'css' : false
   }
 
   return false

packages/tailwindcss-language-service/src/util/find.ts

@@ -4,7 +4,7 @@ import lineColumn from 'line-column'
 import { isCssContext, isCssDoc } from './css'
 import { isHtmlContext } from './html'
 import { isWithinRange } from './isWithinRange'
-import { isJsContext } from './js'
+import { isJsxContext } from './js'
 import { flatten } from './array'
 import { getClassAttributeLexer, getComputedClassAttributeLexer } from './lexers'
 import { getLanguageBoundaries } from './getLanguageBoundaries'
@@ -306,9 +306,13 @@ export async function findClassListsInDocument(
 
   return flatten([
     ...(await Promise.all(
-      boundaries.html.map((range) => findClassListsInHtmlRange(state, doc, range))
+      boundaries
+        .filter((b) => b.type === 'html' || b.type === 'jsx')
+        .map(({ range }) => findClassListsInHtmlRange(state, doc, range))
     )),
-    ...boundaries.css.map((range) => findClassListsInCssRange(doc, range)),
+    ...boundaries
+      .filter((b) => b.type === 'css')
+      .map(({ range }) => findClassListsInCssRange(doc, range)),
     await findCustomClassLists(state, doc),
   ])
 }
@@ -324,7 +328,11 @@ export function findHelperFunctionsInDocument(
   let boundaries = getLanguageBoundaries(state, doc)
   if (!boundaries) return []
 
-  return flatten(boundaries.css.map((range) => findHelperFunctionsInRange(doc, range)))
+  return flatten(
+    boundaries
+      .filter((b) => b.type === 'css')
+      .map(({ range }) => findHelperFunctionsInRange(doc, range))
+  )
 }
 
 export function findHelperFunctionsInRange(
@@ -385,7 +393,7 @@ export async function findClassNameAtPosition(
 
   if (isCssContext(state, doc, position)) {
     classNames = await findClassNamesInRange(state, doc, searchRange, 'css')
-  } else if (isHtmlContext(state, doc, position) || isJsContext(state, doc, position)) {
+  } else if (isHtmlContext(state, doc, position) || isJsxContext(state, doc, position)) {
     classNames = await findClassNamesInRange(state, doc, searchRange, 'html')
   }
 

packages/tailwindcss-language-service/src/util/getLanguageBoundaries.ts

@@ -1,78 +1,153 @@
 import type { TextDocument, Range } from 'vscode-languageserver'
 import { isVueDoc, isHtmlDoc, isSvelteDoc } from './html'
 import { State } from './state'
-import { findAll, indexToPosition } from './find'
+import { indexToPosition } from './find'
 import { isJsDoc } from './js'
+import moo from 'moo'
+import Cache from 'tmp-cache'
 
-export interface LanguageBoundaries {
-  html: Range[]
-  css: Range[]
+export type LanguageBoundary = { type: 'html' | 'js' | 'css' | string; range: Range }
+
+let text = { text: { match: /[^]/, lineBreaks: true } }
+
+let states = {
+  main: {
+    cssBlockStart: { match: '<style', push: 'cssBlock' },
+    jsBlockStart: { match: '<script', push: 'jsBlock' },
+    ...text,
+  },
+  cssBlock: {
+    styleStart: { match: '>', next: 'style' },
+    cssBlockEnd: { match: '/>', pop: 1 },
+    attrStartDouble: { match: '"', push: 'attrDouble' },
+    attrStartSingle: { match: "'", push: 'attrSingle' },
+    interp: { match: '{', push: 'interp' },
+    ...text,
+  },
+  jsBlock: {
+    scriptStart: { match: '>', next: 'script' },
+    jsBlockEnd: { match: '/>', pop: 1 },
+    langAttrStartDouble: { match: 'lang="', push: 'langAttrDouble' },
+    langAttrStartSingle: { match: "lang='", push: 'langAttrSingle' },
+    attrStartDouble: { match: '"', push: 'attrDouble' },
+    attrStartSingle: { match: "'", push: 'attrSingle' },
+    interp: { match: '{', push: 'interp' },
+    ...text,
+  },
+  interp: {
+    interp: { match: '{', push: 'interp' },
+    end: { match: '}', pop: 1 },
+    ...text,
+  },
+  langAttrDouble: {
+    langAttrEnd: { match: '"', pop: 1 },
+    lang: { match: /[^"]+/, lineBreaks: true },
+  },
+  langAttrSingle: {
+    langAttrEnd: { match: "'", pop: 1 },
+    lang: { match: /[^']+/, lineBreaks: true },
+  },
+  attrDouble: {
+    attrEnd: { match: '"', pop: 1 },
+    ...text,
+  },
+  attrSingle: {
+    attrEnd: { match: "'", pop: 1 },
+    ...text,
+  },
+  style: {
+    cssBlockEnd: { match: '</style>', pop: 1 },
+    ...text,
+  },
+  script: {
+    jsBlockEnd: { match: '</script>', pop: 1 },
+    ...text,
+  },
 }
 
-export function getLanguageBoundaries(state: State, doc: TextDocument): LanguageBoundaries | null {
-  if (isVueDoc(doc)) {
-    let text = doc.getText()
-    let blocks = findAll(
-      /(?<open><(?<type>template|style|script)\b[^>]*>).*?(?<close><\/\k<type>>|$)/gis,
-      text
-    )
-    let htmlRanges: Range[] = []
-    let cssRanges: Range[] = []
-    for (let i = 0; i < blocks.length; i++) {
-      let range = {
-        start: indexToPosition(text, blocks[i].index + blocks[i].groups.open.length),
-        end: indexToPosition(
-          text,
-          blocks[i].index + blocks[i][0].length - blocks[i].groups.close.length
-        ),
-      }
-      if (blocks[i].groups.type === 'style') {
-        cssRanges.push(range)
-      } else {
-        htmlRanges.push(range)
-      }
+let vueStates = {
+  ...states,
+  main: {
+    htmlBlockStart: { match: '<template', push: 'htmlBlock' },
+    ...states.main,
+  },
+  htmlBlock: {
+    htmlStart: { match: '>', next: 'html' },
+    htmlBlockEnd: { match: '/>', pop: 1 },
+    attrStartDouble: { match: '"', push: 'attrDouble' },
+    attrStartSingle: { match: "'", push: 'attrSingle' },
+    interp: { match: '{', push: 'interp' },
+    ...text,
+  },
+  html: {
+    htmlBlockEnd: { match: '</template>', pop: 1 },
+    ...text,
+  },
 }
 
-    return {
-      html: htmlRanges,
-      css: cssRanges,
-    }
+let defaultLexer = moo.states(states)
+let vueLexer = moo.states(vueStates)
+
+let cache = new Cache<string, LanguageBoundary[] | null>({ max: 25, maxAge: 1000 })
+
+export function getLanguageBoundaries(
+  state: State,
+  doc: TextDocument,
+  text: string = doc.getText()
+): LanguageBoundary[] | null {
+  let cacheKey = `${doc.languageId}:${text}`
+  if (cache.has(cacheKey)) {
+    return cache.get(cacheKey)
   }
 
-  if (isHtmlDoc(state, doc) || isJsDoc(state, doc) || isSvelteDoc(doc)) {
-    let text = doc.getText()
-    let styleBlocks = findAll(
-      /(?<open><style(?:\s[^>]*[^\/]>|\s*>)).*?(?<close><\/style>|$)/gis,
-      text
-    )
-    let htmlRanges: Range[] = []
-    let cssRanges: Range[] = []
-    let currentIndex = 0
-
-    for (let i = 0; i < styleBlocks.length; i++) {
-      htmlRanges.push({
-        start: indexToPosition(text, currentIndex),
-        end: indexToPosition(text, styleBlocks[i].index),
-      })
-      cssRanges.push({
-        start: indexToPosition(text, styleBlocks[i].index + styleBlocks[i].groups.open.length),
-        end: indexToPosition(
-          text,
-          styleBlocks[i].index + styleBlocks[i][0].length - styleBlocks[i].groups.close.length
-        ),
-      })
-      currentIndex = styleBlocks[i].index + styleBlocks[i][0].length
-    }
-    htmlRanges.push({
-      start: indexToPosition(text, currentIndex),
-      end: indexToPosition(text, text.length),
-    })
-
-    return {
-      html: htmlRanges,
-      css: cssRanges,
-    }
-  }
+  let defaultType = isVueDoc(doc)
+    ? 'none'
+    : isHtmlDoc(state, doc) || isJsDoc(state, doc) || isSvelteDoc(doc)
+    ? 'html'
+    : null
 
+  if (defaultType === null) {
+    cache.set(cacheKey, null)
     return null
   }
+
+  let lexer = defaultType === 'none' ? vueLexer : defaultLexer
+  lexer.reset(text)
+
+  let type = defaultType
+  let boundaries: LanguageBoundary[] = [
+    { type: defaultType, range: { start: { line: 0, character: 0 }, end: undefined } },
+  ]
+  let offset = 0
+
+  try {
+    for (let token of lexer) {
+      if (token.type.endsWith('BlockStart')) {
+        let position = indexToPosition(text, offset)
+        if (!boundaries[boundaries.length - 1].range.end) {
+          boundaries[boundaries.length - 1].range.end = position
+        }
+        type = token.type.replace(/BlockStart$/, '')
+        boundaries.push({ type, range: { start: position, end: undefined } })
+      } else if (token.type.endsWith('BlockEnd')) {
+        let position = indexToPosition(text, offset)
+        boundaries[boundaries.length - 1].range.end = position
+        boundaries.push({ type: defaultType, range: { start: position, end: undefined } })
+      } else if (token.type === 'lang') {
+        boundaries[boundaries.length - 1].type = token.text
+      }
+      offset += token.text.length
+    }
+  } catch {
+    cache.set(cacheKey, null)
+    return null
+  }
+
+  if (!boundaries[boundaries.length - 1].range.end) {
+    boundaries[boundaries.length - 1].range.end = indexToPosition(text, offset)
+  }
+
+  cache.set(cacheKey, boundaries)
+
+  return boundaries
+}

packages/tailwindcss-language-service/src/util/html.ts

@@ -1,6 +1,7 @@
 import type { TextDocument, Position } from 'vscode-languageserver'
 import { State } from './state'
 import { htmlLanguages } from './languages'
+import { getLanguageBoundaries } from './getLanguageBoundaries'
 
 export function isHtmlDoc(state: State, doc: TextDocument): boolean {
   const userHtmlLanguages = Object.keys(state.editor.userLanguages).filter((lang) =>
@@ -24,33 +25,7 @@ export function isHtmlContext(state: State, doc: TextDocument, position: Positio
     end: position,
   })
 
-  if (isHtmlDoc(state, doc) && !isInsideTag(str, ['script', 'style'])) {
-    return true
-  }
+  let boundaries = getLanguageBoundaries(state, doc, str)
 
-  if (isVueDoc(doc)) {
-    return isInsideTag(str, ['template'])
-  }
-
-  if (isSvelteDoc(doc)) {
-    return !isInsideTag(str, ['script', 'style'])
-  }
-
-  return false
-}
-
-export function isInsideTag(str: string, tag: string | string[]): boolean {
-  let open = 0
-  let close = 0
-  let match: RegExpExecArray
-  let tags = Array.isArray(tag) ? tag : [tag]
-  let regex = new RegExp(`<(?<slash>/?)(?:${tags.join('|')})(?:\\s[^>]*[^\/]>|\\s*>)`, 'ig')
-  while ((match = regex.exec(str)) !== null) {
-    if (match.groups.slash) {
-      close += 1
-    } else {
-      open += 1
-    }
-  }
-  return open > 0 && open > close
+  return boundaries ? boundaries[boundaries.length - 1].type === 'html' : false
 }

packages/tailwindcss-language-service/src/util/js.ts

@@ -1,7 +1,7 @@
 import type { TextDocument, Position } from 'vscode-languageserver'
-import { isHtmlDoc, isInsideTag, isVueDoc, isSvelteDoc } from './html'
 import { State } from './state'
 import { jsLanguages } from './languages'
+import { getLanguageBoundaries } from './getLanguageBoundaries'
 
 export function isJsDoc(state: State, doc: TextDocument): boolean {
   const userJsLanguages = Object.keys(state.editor.userLanguages).filter((lang) =>
@@ -11,23 +11,13 @@ export function isJsDoc(state: State, doc: TextDocument): boolean {
   return [...jsLanguages, ...userJsLanguages].indexOf(doc.languageId) !== -1
 }
 
-export function isJsContext(state: State, doc: TextDocument, position: Position): boolean {
-  if (isJsDoc(state, doc)) {
-    return true
-  }
-
+export function isJsxContext(state: State, doc: TextDocument, position: Position): boolean {
   let str = doc.getText({
     start: { line: 0, character: 0 },
     end: position,
   })
 
-  if (isHtmlDoc(state, doc) && isInsideTag(str, ['script'])) {
-    return true
-  }
+  let boundaries = getLanguageBoundaries(state, doc, str)
 
-  if (isVueDoc(doc) || isSvelteDoc(doc)) {
-    return isInsideTag(str, ['script'])
-  }
-
-  return false
+  return boundaries ? boundaries[boundaries.length - 1].type === 'jsx' : false
 }