css-tokenizer and parser-algorithms : fixes and performance improvements (#760)

* css-parser-algorithms : fix unclosed blocks and functions

* add more tests

* one more test

* performance improvements

* improve performance

* wip

* wip

* more performance improvements

* finish up

* improve benchmarks

* little bit faster

* fixes and some convenience methods

* more improvements

* more tests and fixes

* fix

* better error type

* custom media parsing

* fmt

* fix

* add normalization for Simple Block and Function

* more tests

* add css-tokenizer-tests for increased test coverage

* more tests

* update readme's

Author: romainmenke

package-lock.json

@@ -1 +1 @@
-"use strict";var e=require("@csstools/css-tokenizer"),t=require("@csstools/css-parser-algorithms");class LayerName{parts;constructor(e){this.parts=e}tokens(){return[...this.parts]}slice(t,n){const r=[];for(let t=0;t<this.parts.length;t++)this.parts[t][0]===e.TokenType.Ident&&r.push(t);const s=r.slice(t,n);return new LayerName(this.parts.slice(s[0],s[s.length-1]+1))}concat(t){const n=[e.TokenType.Delim,".",-1,-1,{value:"."}];return new LayerName([...this.parts.filter((t=>t[0]===e.TokenType.Ident||t[0]===e.TokenType.Delim)),n,...t.parts.filter((t=>t[0]===e.TokenType.Ident||t[0]===e.TokenType.Delim))])}segments(){return this.parts.filter((t=>t[0]===e.TokenType.Ident)).map((e=>e[4].value))}name(){return this.parts.filter((t=>t[0]===e.TokenType.Ident||t[0]===e.TokenType.Delim)).map((e=>e[1])).join("")}equal(e){const t=this.segments(),n=e.segments();if(t.length!==n.length)return!1;for(let e=0;e<t.length;e++){if(t[e]!==n[e])return!1}return!0}toString(){return e.stringify(...this.parts)}toJSON(){return{parts:this.parts,segments:this.segments(),name:this.name()}}}function parseFromTokens(n,r){const s=t.parseCommaSeparatedListOfComponentValues(n,{onParseError:null==r?void 0:r.onParseError}),o=(null==r?void 0:r.onParseError)??(()=>{}),genericError=e=>({message:`Invalid cascade layer name. ${e}`,start:n[0][2],end:n[n.length-1][3],state:["6.4.2. Layer Naming and Nesting","Layer name syntax","<layer-name> = <ident> [ '.' <ident> ]*"]}),a=[];for(let n=0;n<s.length;n++){const r=s[n];for(let e=0;e<r.length;e++){const n=r[e];if(!t.isTokenNode(n)&&!t.isCommentNode(n)&&!t.isWhitespaceNode(n))return o(genericError(`Invalid layer name part "${n.toString()}"`)),[]}const i=r.flatMap((e=>e.tokens()));let p,l=!1,m=!1;for(let t=0;t<i.length;t++){const n=i[t];if(n[0]!==e.TokenType.Comment&&n[0]!==e.TokenType.Whitespace&&n[0]!==e.TokenType.Ident&&(n[0]!==e.TokenType.Delim||"."!==n[4].value))return o(genericError(`Invalid character "${n[1]}"`)),[];if(!l&&n[0]===e.TokenType.Delim)return o(genericError("Layer names can not start with a dot.")),[];if(l){if(n[0]===e.TokenType.Whitespace){m=!0;continue}if(m&&n[0]===e.TokenType.Comment)continue;if(m)return o(genericError("Encountered unexpected whitespace between layer name parts.")),[];if(p[0]===e.TokenType.Ident&&n[0]===e.TokenType.Ident)return o(genericError("Layer name parts must be separated by dots.")),[];if(p[0]===e.TokenType.Delim&&n[0]===e.TokenType.Delim)return o(genericError("Layer name parts must not be empty.")),[]}n[0]===e.TokenType.Ident&&(l=!0),n[0]!==e.TokenType.Ident&&n[0]!==e.TokenType.Delim||(p=n)}if(!p)return o(genericError("Empty layer name.")),[];if(p[0]===e.TokenType.Delim)return o(genericError("Layer name must not end with a dot.")),[];a.push(new LayerName(i))}return a}exports.LayerName=LayerName,exports.addLayerToModel=function addLayerToModel(e,t){return t.forEach((t=>{const n=t.segments();e:for(let r=0;r<n.length;r++){const n=t.slice(0,r+1),s=n.segments();let o=-1,a=0;for(let t=0;t<e.length;t++){const n=e[t].segments();let r=0;t:for(let e=0;e<n.length;e++){const t=n[e],o=s[e];if(o===t&&e+1===s.length)continue e;if(o!==t){if(o!==t)break t}else r++}r>=a&&(o=t,a=r)}-1===o?e.push(n):e.splice(o+1,0,n)}})),e},exports.parse=function parse(t,n){const r=e.tokenizer({css:t},{commentsAreTokens:!0,onParseError:null==n?void 0:n.onParseError}),s=[];for(;!r.endOfFile();)s.push(r.nextToken());return s.push(r.nextToken()),parseFromTokens(s,n)},exports.parseFromTokens=parseFromTokens;
+"use strict";var e=require("@csstools/css-tokenizer"),n=require("@csstools/css-parser-algorithms");class LayerName{parts;constructor(e){this.parts=e}tokens(){return[...this.parts]}slice(n,r){const t=[];for(let n=0;n<this.parts.length;n++)this.parts[n][0]===e.TokenType.Ident&&t.push(n);const a=t.slice(n,r);return new LayerName(this.parts.slice(a[0],a[a.length-1]+1))}concat(n){const r=[e.TokenType.Delim,".",-1,-1,{value:"."}];return new LayerName([...this.parts.filter((n=>n[0]===e.TokenType.Ident||n[0]===e.TokenType.Delim)),r,...n.parts.filter((n=>n[0]===e.TokenType.Ident||n[0]===e.TokenType.Delim))])}segments(){return this.parts.filter((n=>n[0]===e.TokenType.Ident)).map((e=>e[4].value))}name(){return this.parts.filter((n=>n[0]===e.TokenType.Ident||n[0]===e.TokenType.Delim)).map((e=>e[1])).join("")}equal(e){const n=this.segments(),r=e.segments();if(n.length!==r.length)return!1;for(let e=0;e<n.length;e++){if(n[e]!==r[e])return!1}return!0}toString(){return e.stringify(...this.parts)}toJSON(){return{parts:this.parts,segments:this.segments(),name:this.name()}}}function parseFromTokens(r,t){const a=n.parseCommaSeparatedListOfComponentValues(r,{onParseError:null==t?void 0:t.onParseError}),s=(null==t?void 0:t.onParseError)??(()=>{}),o=["6.4.2. Layer Naming and Nesting","Layer name syntax","<layer-name> = <ident> [ '.' <ident> ]*"],i=r[0][2],l=r[r.length-1][3],p=[];for(let r=0;r<a.length;r++){const t=a[r];for(let r=0;r<t.length;r++){const a=t[r];if(!n.isTokenNode(a)&&!n.isCommentNode(a)&&!n.isWhitespaceNode(a))return s(new e.ParseError(`Invalid cascade layer name. Invalid layer name part "${a.toString()}"`,i,l,o)),[]}const m=t.flatMap((e=>e.tokens()));let c,T=!1,y=!1;for(let n=0;n<m.length;n++){const r=m[n];if(r[0]!==e.TokenType.Comment&&r[0]!==e.TokenType.Whitespace&&r[0]!==e.TokenType.Ident&&(r[0]!==e.TokenType.Delim||"."!==r[4].value))return s(new e.ParseError(`Invalid cascade layer name. Invalid character "${r[1]}"`,i,l,o)),[];if(!T&&r[0]===e.TokenType.Delim)return s(new e.ParseError("Invalid cascade layer name. Layer names can not start with a dot.",i,l,o)),[];if(T){if(r[0]===e.TokenType.Whitespace){y=!0;continue}if(y&&r[0]===e.TokenType.Comment)continue;if(y)return s(new e.ParseError("Invalid cascade layer name. Encountered unexpected whitespace between layer name parts.",i,l,o)),[];if(c[0]===e.TokenType.Ident&&r[0]===e.TokenType.Ident)return s(new e.ParseError("Invalid cascade layer name. Layer name parts must be separated by dots.",i,l,o)),[];if(c[0]===e.TokenType.Delim&&r[0]===e.TokenType.Delim)return s(new e.ParseError("Invalid cascade layer name. Layer name parts must not be empty.",i,l,o)),[]}r[0]===e.TokenType.Ident&&(T=!0),r[0]!==e.TokenType.Ident&&r[0]!==e.TokenType.Delim||(c=r)}if(!c)return s(new e.ParseError("Invalid cascade layer name. Empty layer name.",i,l,o)),[];if(c[0]===e.TokenType.Delim)return s(new e.ParseError("Invalid cascade layer name. Layer name must not end with a dot.",i,l,o)),[];p.push(new LayerName(m))}return p}exports.LayerName=LayerName,exports.addLayerToModel=function addLayerToModel(e,n){return n.forEach((n=>{const r=n.segments();e:for(let t=0;t<r.length;t++){const r=n.slice(0,t+1),a=r.segments();let s=-1,o=0;for(let n=0;n<e.length;n++){const r=e[n].segments();let t=0;n:for(let e=0;e<r.length;e++){const n=r[e],s=a[e];if(s===n&&e+1===a.length)continue e;if(s!==n){if(s!==n)break n}else t++}t>=o&&(s=n,o=t)}-1===s?e.push(r):e.splice(s+1,0,r)}})),e},exports.parse=function parse(n,r){const t=e.tokenizer({css:n},{commentsAreTokens:!0,onParseError:null==r?void 0:r.onParseError}),a=[];for(;!t.endOfFile();)a.push(t.nextToken());return a.push(t.nextToken()),parseFromTokens(a,r)},exports.parseFromTokens=parseFromTokens;

packages/cascade-layer-name-parser/dist/index.cjs

@@ -1 +1 @@
-import{TokenType as e,stringify as t,tokenizer as n}from"@csstools/css-tokenizer";import{parseCommaSeparatedListOfComponentValues as r,isTokenNode as s,isCommentNode as a,isWhitespaceNode as o}from"@csstools/css-parser-algorithms";class LayerName{parts;constructor(e){this.parts=e}tokens(){return[...this.parts]}slice(t,n){const r=[];for(let t=0;t<this.parts.length;t++)this.parts[t][0]===e.Ident&&r.push(t);const s=r.slice(t,n);return new LayerName(this.parts.slice(s[0],s[s.length-1]+1))}concat(t){const n=[e.Delim,".",-1,-1,{value:"."}];return new LayerName([...this.parts.filter((t=>t[0]===e.Ident||t[0]===e.Delim)),n,...t.parts.filter((t=>t[0]===e.Ident||t[0]===e.Delim))])}segments(){return this.parts.filter((t=>t[0]===e.Ident)).map((e=>e[4].value))}name(){return this.parts.filter((t=>t[0]===e.Ident||t[0]===e.Delim)).map((e=>e[1])).join("")}equal(e){const t=this.segments(),n=e.segments();if(t.length!==n.length)return!1;for(let e=0;e<t.length;e++){if(t[e]!==n[e])return!1}return!0}toString(){return t(...this.parts)}toJSON(){return{parts:this.parts,segments:this.segments(),name:this.name()}}}function addLayerToModel(e,t){return t.forEach((t=>{const n=t.segments();e:for(let r=0;r<n.length;r++){const n=t.slice(0,r+1),s=n.segments();let a=-1,o=0;for(let t=0;t<e.length;t++){const n=e[t].segments();let r=0;t:for(let e=0;e<n.length;e++){const t=n[e],a=s[e];if(a===t&&e+1===s.length)continue e;if(a!==t){if(a!==t)break t}else r++}r>=o&&(a=t,o=r)}-1===a?e.push(n):e.splice(a+1,0,n)}})),e}function parseFromTokens(t,n){const i=r(t,{onParseError:null==n?void 0:n.onParseError}),l=(null==n?void 0:n.onParseError)??(()=>{}),genericError=e=>({message:`Invalid cascade layer name. ${e}`,start:t[0][2],end:t[t.length-1][3],state:["6.4.2. Layer Naming and Nesting","Layer name syntax","<layer-name> = <ident> [ '.' <ident> ]*"]}),m=[];for(let t=0;t<i.length;t++){const n=i[t];for(let e=0;e<n.length;e++){const t=n[e];if(!s(t)&&!a(t)&&!o(t))return l(genericError(`Invalid layer name part "${t.toString()}"`)),[]}const r=n.flatMap((e=>e.tokens()));let c,u=!1,p=!1;for(let t=0;t<r.length;t++){const n=r[t];if(n[0]!==e.Comment&&n[0]!==e.Whitespace&&n[0]!==e.Ident&&(n[0]!==e.Delim||"."!==n[4].value))return l(genericError(`Invalid character "${n[1]}"`)),[];if(!u&&n[0]===e.Delim)return l(genericError("Layer names can not start with a dot.")),[];if(u){if(n[0]===e.Whitespace){p=!0;continue}if(p&&n[0]===e.Comment)continue;if(p)return l(genericError("Encountered unexpected whitespace between layer name parts.")),[];if(c[0]===e.Ident&&n[0]===e.Ident)return l(genericError("Layer name parts must be separated by dots.")),[];if(c[0]===e.Delim&&n[0]===e.Delim)return l(genericError("Layer name parts must not be empty.")),[]}n[0]===e.Ident&&(u=!0),n[0]!==e.Ident&&n[0]!==e.Delim||(c=n)}if(!c)return l(genericError("Empty layer name.")),[];if(c[0]===e.Delim)return l(genericError("Layer name must not end with a dot.")),[];m.push(new LayerName(r))}return m}function parse(e,t){const r=n({css:e},{commentsAreTokens:!0,onParseError:null==t?void 0:t.onParseError}),s=[];for(;!r.endOfFile();)s.push(r.nextToken());return s.push(r.nextToken()),parseFromTokens(s,t)}export{LayerName,addLayerToModel,parse,parseFromTokens};
+import{TokenType as e,stringify as t,ParseError as n,tokenizer as r}from"@csstools/css-tokenizer";import{parseCommaSeparatedListOfComponentValues as a,isTokenNode as s,isCommentNode as o,isWhitespaceNode as i}from"@csstools/css-parser-algorithms";class LayerName{parts;constructor(e){this.parts=e}tokens(){return[...this.parts]}slice(t,n){const r=[];for(let t=0;t<this.parts.length;t++)this.parts[t][0]===e.Ident&&r.push(t);const a=r.slice(t,n);return new LayerName(this.parts.slice(a[0],a[a.length-1]+1))}concat(t){const n=[e.Delim,".",-1,-1,{value:"."}];return new LayerName([...this.parts.filter((t=>t[0]===e.Ident||t[0]===e.Delim)),n,...t.parts.filter((t=>t[0]===e.Ident||t[0]===e.Delim))])}segments(){return this.parts.filter((t=>t[0]===e.Ident)).map((e=>e[4].value))}name(){return this.parts.filter((t=>t[0]===e.Ident||t[0]===e.Delim)).map((e=>e[1])).join("")}equal(e){const t=this.segments(),n=e.segments();if(t.length!==n.length)return!1;for(let e=0;e<t.length;e++){if(t[e]!==n[e])return!1}return!0}toString(){return t(...this.parts)}toJSON(){return{parts:this.parts,segments:this.segments(),name:this.name()}}}function addLayerToModel(e,t){return t.forEach((t=>{const n=t.segments();e:for(let r=0;r<n.length;r++){const n=t.slice(0,r+1),a=n.segments();let s=-1,o=0;for(let t=0;t<e.length;t++){const n=e[t].segments();let r=0;t:for(let e=0;e<n.length;e++){const t=n[e],s=a[e];if(s===t&&e+1===a.length)continue e;if(s!==t){if(s!==t)break t}else r++}r>=o&&(s=t,o=r)}-1===s?e.push(n):e.splice(s+1,0,n)}})),e}function parseFromTokens(t,r){const l=a(t,{onParseError:null==r?void 0:r.onParseError}),m=(null==r?void 0:r.onParseError)??(()=>{}),c=["6.4.2. Layer Naming and Nesting","Layer name syntax","<layer-name> = <ident> [ '.' <ident> ]*"],d=t[0][2],u=t[t.length-1][3],p=[];for(let t=0;t<l.length;t++){const r=l[t];for(let e=0;e<r.length;e++){const t=r[e];if(!s(t)&&!o(t)&&!i(t))return m(new n(`Invalid cascade layer name. Invalid layer name part "${t.toString()}"`,d,u,c)),[]}const a=r.flatMap((e=>e.tokens()));let h,f=!1,y=!1;for(let t=0;t<a.length;t++){const r=a[t];if(r[0]!==e.Comment&&r[0]!==e.Whitespace&&r[0]!==e.Ident&&(r[0]!==e.Delim||"."!==r[4].value))return m(new n(`Invalid cascade layer name. Invalid character "${r[1]}"`,d,u,c)),[];if(!f&&r[0]===e.Delim)return m(new n("Invalid cascade layer name. Layer names can not start with a dot.",d,u,c)),[];if(f){if(r[0]===e.Whitespace){y=!0;continue}if(y&&r[0]===e.Comment)continue;if(y)return m(new n("Invalid cascade layer name. Encountered unexpected whitespace between layer name parts.",d,u,c)),[];if(h[0]===e.Ident&&r[0]===e.Ident)return m(new n("Invalid cascade layer name. Layer name parts must be separated by dots.",d,u,c)),[];if(h[0]===e.Delim&&r[0]===e.Delim)return m(new n("Invalid cascade layer name. Layer name parts must not be empty.",d,u,c)),[]}r[0]===e.Ident&&(f=!0),r[0]!==e.Ident&&r[0]!==e.Delim||(h=r)}if(!h)return m(new n("Invalid cascade layer name. Empty layer name.",d,u,c)),[];if(h[0]===e.Delim)return m(new n("Invalid cascade layer name. Layer name must not end with a dot.",d,u,c)),[];p.push(new LayerName(a))}return p}function parse(e,t){const n=r({css:e},{commentsAreTokens:!0,onParseError:null==t?void 0:t.onParseError}),a=[];for(;!n.endOfFile();)a.push(n.nextToken());return a.push(n.nextToken()),parseFromTokens(a,t)}export{LayerName,addLayerToModel,parse,parseFromTokens};

packages/cascade-layer-name-parser/dist/index.mjs

@@ -1,8 +1,7 @@
-import { ParserError } from '@csstools/css-parser-algorithms/dist/interfaces/error';
-import { CSSToken } from '@csstools/css-tokenizer';
+import { CSSToken, ParseError } from '@csstools/css-tokenizer';
 import { LayerName } from '../nodes/layer-name';
 export type Options = {
-    onParseError?: (error: ParserError) => void;
+    onParseError?: (error: ParseError) => void;
 };
 export declare function parseFromTokens(tokens: Array<CSSToken>, options?: Options): LayerName[];
 export declare function parse(source: string, options?: Options): LayerName[];