From 9de1eb081cd41a20a0e66e6fb37dd17a583d6bde Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 11 Apr 2020 23:57:46 +0100 Subject: [PATCH 01/14] Use a new "gitea" frontmatter section to control rendering * Add control for the rendering of the frontmatter * Add control to include a TOC Signed-off-by: Andrew Thornton --- go.mod | 1 + modules/markup/html.go | 40 +++++++ modules/markup/markdown/ast.go | 107 +++++++++++++++++ modules/markup/markdown/goldmark.go | 128 ++++++++++++++++++-- modules/markup/markdown/markdown.go | 4 +- modules/markup/markdown/renderconfig.go | 149 ++++++++++++++++++++++++ modules/markup/markdown/toc.go | 47 ++++++++ modules/markup/sanitizer.go | 3 + vendor/modules.txt | 1 + 9 files changed, 469 insertions(+), 11 deletions(-) create mode 100644 modules/markup/markdown/ast.go create mode 100644 modules/markup/markdown/renderconfig.go create mode 100644 modules/markup/markdown/toc.go diff --git a/go.mod b/go.mod index 0930b0d168a9b..3d5622a8c8f8f 100644 --- a/go.mod +++ b/go.mod @@ -120,6 +120,7 @@ require ( gopkg.in/ini.v1 v1.52.0 gopkg.in/ldap.v3 v3.0.2 gopkg.in/testfixtures.v2 v2.5.0 + gopkg.in/yaml.v2 v2.2.4 mvdan.cc/xurls/v2 v2.1.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 xorm.io/builder v0.3.7 diff --git a/modules/markup/html.go b/modules/markup/html.go index 51d161ecca827..a179385243316 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -351,6 +351,46 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { visitText = false } else if node.Data == "code" || node.Data == "pre" { return + } else if node.Data == "i" { + for _, attr := range node.Attr { + if attr.Key != "class" { + continue + } + // Got a class + from := 0 + + for idx := strings.Index(attr.Val[from:], "icon") + from; idx >= from; { + if (idx != 0 && attr.Val[idx-1] != ' ') || + (len(attr.Val) > idx+4 && attr.Val[idx+4] != ' ') { + from = idx + 4 + continue + } + // We should be an icon + end := idx + 4 + + // now need to move the icon class first... + if idx != 0 && attr.Val[idx-1] == ' ' { + idx-- + } + class := "icon " + attr.Val[:idx] + if len(attr.Val) > end+1 && attr.Val[end] == ' ' { + class += " " + attr.Val[end:] + } + attr.Val = class + + // Remove all children of icons + child := node.FirstChild + for child != nil { + node.RemoveChild(child) + child = child.NextSibling + } + node.FirstChild = nil + node.LastChild = nil + + break + } + break + } } for n := node.FirstChild; n != nil; n = n.NextSibling { ctx.visitNode(n, visitText) diff --git a/modules/markup/markdown/ast.go b/modules/markup/markdown/ast.go new file mode 100644 index 0000000000000..f79d12435b431 --- /dev/null +++ b/modules/markup/markdown/ast.go @@ -0,0 +1,107 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package markdown + +import "github.com/yuin/goldmark/ast" + +// Details is a block that contains Summary and details +type Details struct { + ast.BaseBlock +} + +// Dump implements Node.Dump . +func (n *Details) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// KindDetails is the NodeKind for Details +var KindDetails = ast.NewNodeKind("Details") + +// Kind implements Node.Kind. +func (n *Details) Kind() ast.NodeKind { + return KindDetails +} + +// NewDetails returns a new Paragraph node. +func NewDetails() *Details { + return &Details{ + BaseBlock: ast.BaseBlock{}, + } +} + +// IsDetails returns true if the given node implements the Details interface, +// otherwise false. +func IsDetails(node ast.Node) bool { + _, ok := node.(*Details) + return ok +} + +// Summary is a block that contains the summary of details block +type Summary struct { + ast.BaseBlock +} + +// Dump implements Node.Dump . +func (n *Summary) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// KindSummary is the NodeKind for Summary +var KindSummary = ast.NewNodeKind("Summary") + +// Kind implements Node.Kind. +func (n *Summary) Kind() ast.NodeKind { + return KindSummary +} + +// NewSummary returns a new Summary node. +func NewSummary() *Summary { + return &Summary{ + BaseBlock: ast.BaseBlock{}, + } +} + +// IsSummary returns true if the given node implements the Summary interface, +// otherwise false. +func IsSummary(node ast.Node) bool { + _, ok := node.(*Summary) + return ok +} + +// Icon is an inline for a fomantic icon +type Icon struct { + ast.BaseInline + Name []byte +} + +// Dump implements Node.Dump . +func (n *Icon) Dump(source []byte, level int) { + m := map[string]string{} + m["Name"] = string(n.Name) + ast.DumpHelper(n, source, level, m, nil) +} + +// KindIcon is the NodeKind for Icon +var KindIcon = ast.NewNodeKind("Icon") + +// Kind implements Node.Kind. +func (n *Icon) Kind() ast.NodeKind { + return KindIcon +} + +// NewIcon returns a new Paragraph node. +func NewIcon(name string) *Icon { + return &Icon{ + BaseInline: ast.BaseInline{}, + Name: []byte(name), + } +} + +// IsIcon returns true if the given node implements the Icon interface, +// otherwise false. +func IsIcon(node ast.Node) bool { + _, ok := node.(*Icon) + return ok +} diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 70f47e289eac2..de0a1a326f2e3 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -7,12 +7,15 @@ package markdown import ( "bytes" "fmt" + "regexp" "strings" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/common" giteautil "code.gitea.io/gitea/modules/util" + meta "github.com/yuin/goldmark-meta" "github.com/yuin/goldmark/ast" east "github.com/yuin/goldmark/extension/ast" "github.com/yuin/goldmark/parser" @@ -24,17 +27,54 @@ import ( var byteMailto = []byte("mailto:") +// Header holds the data about a header. +type Header struct { + Level int + Text string + ID string +} + // GiteaASTTransformer is a default transformer of the goldmark tree. type GiteaASTTransformer struct{} // Transform transforms the given AST tree. func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + metaData := meta.GetItems(pc) + firstChild := node.FirstChild() + createTOC := false + var toc = []Header{} + if metaData != nil { + rc := ToRenderConfig(metaData) + log.Info("%v", rc) + + metaNode := rc.toMetaNode(metaData) + if metaNode != nil { + node.InsertBefore(node, firstChild, metaNode) + } + createTOC = rc.TOC + toc = make([]Header, 0, 100) + } + _ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } switch v := n.(type) { + case *ast.Heading: + if createTOC { + log.Info("CreateToc") + text := n.Text(reader.Source()) + header := Header{ + Text: util.BytesToReadOnlyString(text), + Level: v.Level, + } + if id, found := v.AttributeString("id"); found { + header.ID = util.BytesToReadOnlyString(id.([]byte)) + } + toc = append(toc, header) + log.Info("CreateToc: %v", header) + } case *ast.Image: // Images need two things: // @@ -91,6 +131,13 @@ func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, } return ast.WalkContinue, nil }) + + if createTOC && len(toc) > 0 { + tocNode := createTOCNode(toc) + if tocNode != nil { + node.InsertBefore(node, firstChild, tocNode) + } + } } type prefixedIDs struct { @@ -139,10 +186,10 @@ func newPrefixedIDs() *prefixedIDs { } } -// NewTaskCheckBoxHTMLRenderer creates a TaskCheckBoxHTMLRenderer to render tasklists +// NewGiteaHTMLRenderer creates a GiteaHTMLRenderer to render // in the gitea form. -func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { - r := &TaskCheckBoxHTMLRenderer{ +func NewGiteaHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &GiteaHTMLRenderer{ Config: html.NewConfig(), } for _, opt := range opts { @@ -151,19 +198,82 @@ func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { return r } -// TaskCheckBoxHTMLRenderer is a renderer.NodeRenderer implementation that -// renders checkboxes in list items. -// Overrides the default goldmark one to present the gitea format -type TaskCheckBoxHTMLRenderer struct { +// GiteaHTMLRenderer is a renderer.NodeRenderer implementation that +// renders gitea specific features. +type GiteaHTMLRenderer struct { html.Config } // RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. -func (r *TaskCheckBoxHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { +func (r *GiteaHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(KindDetails, r.renderDetails) + reg.Register(KindSummary, r.renderSummary) + reg.Register(KindIcon, r.renderIcon) reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) } -func (r *TaskCheckBoxHTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *GiteaHTMLRenderer) renderDetails(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + var err error + if entering { + _, err = w.WriteString("
") + } else { + _, err = w.WriteString("
") + } + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + +func (r *GiteaHTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + var err error + if entering { + _, err = w.WriteString("") + } else { + _, err = w.WriteString("") + } + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + +var validNameRE = regexp.MustCompile("^[a-z ]+$") + +func (r *GiteaHTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + n := node.(*Icon) + + name := strings.TrimSpace(strings.ToLower(string(n.Name))) + + if len(name) == 0 { + // skip this + return ast.WalkContinue, nil + } + + if !validNameRE.MatchString(name) { + // skip this + return ast.WalkContinue, nil + } + + var err error + _, err = w.WriteString(fmt.Sprintf(``, name)) + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + +func (r *GiteaHTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index c48bbab301003..af493ebe39c46 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -54,7 +54,7 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { extension.Ellipsis: nil, }), ), - meta.New(meta.WithTable()), + meta.Meta, ), goldmark.WithParserOptions( parser.WithAttribute(), @@ -71,7 +71,7 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { // Override the original Tasklist renderer! converter.Renderer().AddOptions( renderer.WithNodeRenderers( - util.Prioritized(NewTaskCheckBoxHTMLRenderer(), 1000), + util.Prioritized(NewGiteaHTMLRenderer(), 1000), ), ) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go new file mode 100644 index 0000000000000..c0568391a0ca6 --- /dev/null +++ b/modules/markup/markdown/renderconfig.go @@ -0,0 +1,149 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package markdown + +import ( + "fmt" + "strings" + + "github.com/yuin/goldmark/ast" + east "github.com/yuin/goldmark/extension/ast" + "gopkg.in/yaml.v2" +) + +// RenderConfig represents rendering configuration for this file +type RenderConfig struct { + Meta string + Icon string + TOC bool +} + +// ToRenderConfig converts a yaml.MapSlice to a RenderConfig +func ToRenderConfig(meta yaml.MapSlice) *RenderConfig { + rc := &RenderConfig{ + Meta: "table", + Icon: "table", + } + if meta == nil { + return rc + } + var giteaMetaControl *yaml.MapItem + for _, item := range meta { + strKey, ok := item.Key.(string) + if !ok { + continue + } + strKey = strings.TrimSpace(strings.ToLower(strKey)) + switch strKey { + case "gitea": + giteaMetaControl = &item + case "include_toc": + val, ok := item.Value.(bool) + if !ok { + continue + } + rc.TOC = val + } + } + + if giteaMetaControl != nil { + switch v := giteaMetaControl.Value.(type) { + case string: + switch v { + case "none": + rc.Meta = "none" + case "table": + rc.Meta = "table" + case "details": + rc.Meta = "details" + default: + rc.Meta = "details" + } + case yaml.MapSlice: + for _, item := range v { + strKey, ok := item.Key.(string) + if !ok { + continue + } + strKey = strings.TrimSpace(strings.ToLower(strKey)) + strValue, ok := item.Value.(string) + strValue = strings.TrimSpace(strings.ToLower(strValue)) + switch strKey { + case "meta": + if !ok { + continue + } + switch strValue { + case "none": + rc.Meta = "none" + case "table": + rc.Meta = "table" + case "details": + rc.Meta = "details" + default: + rc.Meta = "details" + } + case "details_icon": + if !ok { + continue + } + rc.Icon = strValue + case "include_toc": + val, ok := item.Value.(bool) + if !ok { + continue + } + rc.TOC = val + } + } + } + } + return rc +} + +func (rc *RenderConfig) toMetaNode(meta yaml.MapSlice) ast.Node { + switch rc.Meta { + case "table": + return metaToTable(meta) + case "details": + return metaToDetails(meta, rc.Icon) + default: + return nil + } +} + +func metaToTable(meta yaml.MapSlice) ast.Node { + table := east.NewTable() + alignments := []east.Alignment{} + for range meta { + alignments = append(alignments, east.AlignNone) + } + row := east.NewTableRow(alignments) + for _, item := range meta { + cell := east.NewTableCell() + cell.AppendChild(cell, ast.NewString([]byte(fmt.Sprintf("%v", item.Key)))) + row.AppendChild(row, cell) + } + table.AppendChild(table, east.NewTableHeader(row)) + + row = east.NewTableRow(alignments) + for _, item := range meta { + cell := east.NewTableCell() + cell.AppendChild(cell, ast.NewString([]byte(fmt.Sprintf("%v", item.Value)))) + row.AppendChild(row, cell) + } + table.AppendChild(table, row) + return table +} + +func metaToDetails(meta yaml.MapSlice, icon string) ast.Node { + details := NewDetails() + summary := NewSummary() + summary.AppendChild(summary, NewIcon(icon)) + details.AppendChild(details, summary) + details.AppendChild(details, metaToTable(meta)) + + return details +} diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go new file mode 100644 index 0000000000000..c55e480ce06ce --- /dev/null +++ b/modules/markup/markdown/toc.go @@ -0,0 +1,47 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package markdown + +import ( + "fmt" + "net/url" + + "github.com/yuin/goldmark/ast" +) + +func createTOCNode(toc []Header) ast.Node { + details := NewDetails() + summary := NewSummary() + summary.AppendChild(summary, ast.NewString([]byte("Table of Contents"))) + details.AppendChild(details, summary) + ul := ast.NewList('-') + details.AppendChild(details, ul) + currentLevel := 6 + for _, header := range toc { + if header.Level < currentLevel { + currentLevel = header.Level + } + } + for _, header := range toc { + if header.Level < currentLevel { + ul = ul.Parent().(*ast.List) + } else if header.Level > currentLevel { + for currentLevel < header.Level { + newL := ast.NewList('-') + ul.AppendChild(ul, newL) + currentLevel++ + ul = newL + } + } + li := ast.NewListItem(currentLevel * 2) + a := ast.NewLink() + a.Destination = []byte(fmt.Sprintf("#%s", url.PathEscape(header.ID))) + a.AppendChild(a, ast.NewString([]byte(header.Text))) + li.AppendChild(li, a) + ul.AppendChild(ul, li) + } + + return details +} diff --git a/modules/markup/sanitizer.go b/modules/markup/sanitizer.go index b5c6dc25f4f15..95c6eb0dc4d0a 100644 --- a/modules/markup/sanitizer.go +++ b/modules/markup/sanitizer.go @@ -56,6 +56,9 @@ func ReplaceSanitizer() { // Allow classes for task lists sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`task-list`)).OnElements("ul") + // Allow icons + sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^icon(\s+[\p{L}\p{N}_-]+)+$`)).OnElements("i", "span") + // Allow generally safe attributes generalSafeAttrs := []string{"abbr", "accept", "accept-charset", "accesskey", "action", "align", "alt", diff --git a/vendor/modules.txt b/vendor/modules.txt index 8a7c6706e9ee6..9cc54a289a017 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -804,6 +804,7 @@ gopkg.in/toqueteos/substring.v1 # gopkg.in/warnings.v0 v0.1.2 gopkg.in/warnings.v0 # gopkg.in/yaml.v2 v2.2.4 +## explicit gopkg.in/yaml.v2 # mvdan.cc/xurls/v2 v2.1.0 ## explicit From 9865bdc66f619cc7c9bf522d1353aa4dbe595df6 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 12 Apr 2020 09:42:02 +0100 Subject: [PATCH 02/14] As per @lunny Signed-off-by: Andrew Thornton --- modules/markup/markdown/goldmark.go | 28 +++++++++++++--------------- modules/markup/markdown/markdown.go | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index de0a1a326f2e3..dee1066560989 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -34,11 +34,11 @@ type Header struct { ID string } -// GiteaASTTransformer is a default transformer of the goldmark tree. -type GiteaASTTransformer struct{} +// ASTTransformer is a default transformer of the goldmark tree. +type ASTTransformer struct{} // Transform transforms the given AST tree. -func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { +func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { metaData := meta.GetItems(pc) firstChild := node.FirstChild() createTOC := false @@ -63,7 +63,6 @@ func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, switch v := n.(type) { case *ast.Heading: if createTOC { - log.Info("CreateToc") text := n.Text(reader.Source()) header := Header{ Text: util.BytesToReadOnlyString(text), @@ -73,7 +72,6 @@ func (g *GiteaASTTransformer) Transform(node *ast.Document, reader text.Reader, header.ID = util.BytesToReadOnlyString(id.([]byte)) } toc = append(toc, header) - log.Info("CreateToc: %v", header) } case *ast.Image: // Images need two things: @@ -186,10 +184,10 @@ func newPrefixedIDs() *prefixedIDs { } } -// NewGiteaHTMLRenderer creates a GiteaHTMLRenderer to render +// NewHTMLRenderer creates a HTMLRenderer to render // in the gitea form. -func NewGiteaHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { - r := &GiteaHTMLRenderer{ +func NewHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &HTMLRenderer{ Config: html.NewConfig(), } for _, opt := range opts { @@ -198,21 +196,21 @@ func NewGiteaHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { return r } -// GiteaHTMLRenderer is a renderer.NodeRenderer implementation that +// HTMLRenderer is a renderer.NodeRenderer implementation that // renders gitea specific features. -type GiteaHTMLRenderer struct { +type HTMLRenderer struct { html.Config } // RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. -func (r *GiteaHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { +func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { reg.Register(KindDetails, r.renderDetails) reg.Register(KindSummary, r.renderSummary) reg.Register(KindIcon, r.renderIcon) reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) } -func (r *GiteaHTMLRenderer) renderDetails(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderDetails(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { var err error if entering { _, err = w.WriteString("
") @@ -227,7 +225,7 @@ func (r *GiteaHTMLRenderer) renderDetails(w util.BufWriter, source []byte, node return ast.WalkContinue, nil } -func (r *GiteaHTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderSummary(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { var err error if entering { _, err = w.WriteString("") @@ -244,7 +242,7 @@ func (r *GiteaHTMLRenderer) renderSummary(w util.BufWriter, source []byte, node var validNameRE = regexp.MustCompile("^[a-z ]+$") -func (r *GiteaHTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } @@ -273,7 +271,7 @@ func (r *GiteaHTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast return ast.WalkContinue, nil } -func (r *GiteaHTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { +func (r *HTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index af493ebe39c46..e74936e2bdebb 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -71,7 +71,7 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { // Override the original Tasklist renderer! converter.Renderer().AddOptions( renderer.WithNodeRenderers( - util.Prioritized(NewGiteaHTMLRenderer(), 1000), + util.Prioritized(NewHTMLRenderer(), 1000), ), ) From 67ff9011536fe5d97869225de6f24b52fe41ad21 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 12 Apr 2020 13:31:55 +0100 Subject: [PATCH 03/14] Add language control for TOC Signed-off-by: Andrew Thornton --- modules/markup/markdown/goldmark.go | 44 +++++++++++++++++++++-- modules/markup/markdown/markdown.go | 5 ++- modules/markup/markdown/renderconfig.go | 47 +++++++++++++++++-------- modules/markup/markdown/toc.go | 6 ++-- options/locale/locale_en-US.ini | 1 + options/locale/locale_es-ES.ini | 1 + 6 files changed, 82 insertions(+), 22 deletions(-) diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index dee1066560989..59bc3d2c3ca84 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/common" + "code.gitea.io/gitea/modules/setting" giteautil "code.gitea.io/gitea/modules/util" meta "github.com/yuin/goldmark-meta" @@ -43,9 +44,13 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa firstChild := node.FirstChild() createTOC := false var toc = []Header{} + rc := &RenderConfig{ + Meta: "table", + Icon: "table", + Lang: "", + } if metaData != nil { - rc := ToRenderConfig(metaData) - log.Info("%v", rc) + rc.ToRenderConfig(metaData) metaNode := rc.toMetaNode(metaData) if metaNode != nil { @@ -131,11 +136,19 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa }) if createTOC && len(toc) > 0 { - tocNode := createTOCNode(toc) + lang := rc.Lang + if len(lang) == 0 { + lang = setting.Langs[0] + } + tocNode := createTOCNode(toc, rc.Lang) if tocNode != nil { node.InsertBefore(node, firstChild, tocNode) } } + + if len(rc.Lang) > 0 { + node.SetAttributeString("lang", []byte(rc.Lang)) + } } type prefixedIDs struct { @@ -204,12 +217,37 @@ type HTMLRenderer struct { // RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindDocument, r.renderDocument) reg.Register(KindDetails, r.renderDetails) reg.Register(KindSummary, r.renderSummary) reg.Register(KindIcon, r.renderIcon) reg.Register(east.KindTaskCheckBox, r.renderTaskCheckBox) } +func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + log.Info("renderDocument %v", node) + n := node.(*ast.Document) + + var err error + if entering { + _, err = w.WriteString("') + } + } else { + _, err = w.WriteString("") + } + + if err != nil { + return ast.WalkStop, err + } + + return ast.WalkContinue, nil +} + func (r *HTMLRenderer) renderDetails(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { var err error if entering { diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index e74936e2bdebb..e50301ffe4a04 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -60,7 +60,7 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { parser.WithAttribute(), parser.WithAutoHeadingID(), parser.WithASTTransformers( - util.Prioritized(&GiteaASTTransformer{}, 10000), + util.Prioritized(&ASTTransformer{}, 10000), ), ), goldmark.WithRendererOptions( @@ -71,7 +71,7 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { // Override the original Tasklist renderer! converter.Renderer().AddOptions( renderer.WithNodeRenderers( - util.Prioritized(NewHTMLRenderer(), 1000), + util.Prioritized(NewHTMLRenderer(), 10), ), ) @@ -85,7 +85,6 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte { if err := converter.Convert(giteautil.NormalizeEOL(body), &buf, parser.WithContext(pc)); err != nil { log.Error("Unable to render: %v", err) } - return markup.SanitizeReader(&buf).Bytes() } diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index c0568391a0ca6..a136052e9c367 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -18,18 +18,16 @@ type RenderConfig struct { Meta string Icon string TOC bool + Lang string } // ToRenderConfig converts a yaml.MapSlice to a RenderConfig -func ToRenderConfig(meta yaml.MapSlice) *RenderConfig { - rc := &RenderConfig{ - Meta: "table", - Icon: "table", - } +func (rc *RenderConfig) ToRenderConfig(meta yaml.MapSlice) { if meta == nil { - return rc + return } - var giteaMetaControl *yaml.MapItem + found := false + var giteaMetaControl yaml.MapItem for _, item := range meta { strKey, ok := item.Key.(string) if !ok { @@ -38,17 +36,28 @@ func ToRenderConfig(meta yaml.MapSlice) *RenderConfig { strKey = strings.TrimSpace(strings.ToLower(strKey)) switch strKey { case "gitea": - giteaMetaControl = &item + giteaMetaControl = item + found = true case "include_toc": val, ok := item.Value.(bool) if !ok { continue } rc.TOC = val + case "lang": + val, ok := item.Value.(string) + if !ok { + continue + } + val = strings.TrimSpace(val) + if len(val) == 0 { + continue + } + rc.Lang = val } } - if giteaMetaControl != nil { + if found { switch v := giteaMetaControl.Value.(type) { case string: switch v { @@ -68,14 +77,13 @@ func ToRenderConfig(meta yaml.MapSlice) *RenderConfig { continue } strKey = strings.TrimSpace(strings.ToLower(strKey)) - strValue, ok := item.Value.(string) - strValue = strings.TrimSpace(strings.ToLower(strValue)) switch strKey { case "meta": + val, ok := item.Value.(string) if !ok { continue } - switch strValue { + switch strings.TrimSpace(strings.ToLower(val)) { case "none": rc.Meta = "none" case "table": @@ -86,21 +94,32 @@ func ToRenderConfig(meta yaml.MapSlice) *RenderConfig { rc.Meta = "details" } case "details_icon": + val, ok := item.Value.(string) if !ok { continue } - rc.Icon = strValue + rc.Icon = strings.TrimSpace(strings.ToLower(val)) case "include_toc": val, ok := item.Value.(bool) if !ok { continue } rc.TOC = val + case "lang": + val, ok := item.Value.(string) + if !ok { + continue + } + val = strings.TrimSpace(val) + if len(val) == 0 { + continue + } + rc.Lang = val } } } } - return rc + return } func (rc *RenderConfig) toMetaNode(meta yaml.MapSlice) ast.Node { diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go index c55e480ce06ce..d1476747e2559 100644 --- a/modules/markup/markdown/toc.go +++ b/modules/markup/markdown/toc.go @@ -8,13 +8,15 @@ import ( "fmt" "net/url" + "github.com/unknwon/i18n" "github.com/yuin/goldmark/ast" ) -func createTOCNode(toc []Header) ast.Node { +func createTOCNode(toc []Header, lang string) ast.Node { details := NewDetails() summary := NewSummary() - summary.AppendChild(summary, ast.NewString([]byte("Table of Contents"))) + + summary.AppendChild(summary, ast.NewString([]byte(i18n.Tr(lang, "toc")))) details.AppendChild(details, summary) ul := ast.NewList('-') details.AppendChild(details, ul) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index dfdae8f82b691..4d163319818de 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -19,6 +19,7 @@ create_new = Create… user_profile_and_more = Profile and Settings… signed_in_as = Signed in as enable_javascript = This website works better with JavaScript. +toc = Table of Contents username = Username email = Email Address diff --git a/options/locale/locale_es-ES.ini b/options/locale/locale_es-ES.ini index 8e31d773d1ffa..c4eca8345e66f 100644 --- a/options/locale/locale_es-ES.ini +++ b/options/locale/locale_es-ES.ini @@ -19,6 +19,7 @@ create_new=Crear… user_profile_and_more=Perfil y ajustes… signed_in_as=Identificado como enable_javascript=Este sitio web funciona mejor con JavaScript. +toc=Tabla de contenido username=Nombre de usuario email=Correo electrónico From 314ac5491f06ace4f2b8842df52afa8c7b83855a Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 12 Apr 2020 14:06:13 +0100 Subject: [PATCH 04/14] placate lint Signed-off-by: Andrew Thornton --- modules/markup/markdown/goldmark.go | 2 +- modules/markup/markdown/renderconfig.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 59bc3d2c3ca84..280f5928f4d89 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -140,7 +140,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa if len(lang) == 0 { lang = setting.Langs[0] } - tocNode := createTOCNode(toc, rc.Lang) + tocNode := createTOCNode(toc, lang) if tocNode != nil { node.InsertBefore(node, firstChild, tocNode) } diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index a136052e9c367..7f479d650d2c5 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -119,7 +119,6 @@ func (rc *RenderConfig) ToRenderConfig(meta yaml.MapSlice) { } } } - return } func (rc *RenderConfig) toMetaNode(meta yaml.MapSlice) ast.Node { From b87baff0738ce38f74b719c0423c52d9699bdb32 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 12 Apr 2020 21:45:57 +0100 Subject: [PATCH 05/14] placate unit tests Signed-off-by: Andrew Thornton --- modules/markup/markdown/goldmark.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 280f5928f4d89..6edb3e6971953 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -228,21 +228,23 @@ func (r *HTMLRenderer) renderDocument(w util.BufWriter, source []byte, node ast. log.Info("renderDocument %v", node) n := node.(*ast.Document) - var err error - if entering { - _, err = w.WriteString("') + if val, has := n.AttributeString("lang"); has { + var err error + if entering { + _, err = w.WriteString("') + } + } else { + _, err = w.WriteString("") } - } else { - _, err = w.WriteString("") - } - if err != nil { - return ast.WalkStop, err + if err != nil { + return ast.WalkStop, err + } } return ast.WalkContinue, nil From b905baec29958349c6c4757127a81c81be266289 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 18 Apr 2020 17:16:44 +0100 Subject: [PATCH 06/14] Remove mistaken added translation --- options/locale/locale_es-ES.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/options/locale/locale_es-ES.ini b/options/locale/locale_es-ES.ini index 7b8da94ccea0e..656c150c9b332 100644 --- a/options/locale/locale_es-ES.ini +++ b/options/locale/locale_es-ES.ini @@ -19,7 +19,6 @@ create_new=Crear… user_profile_and_more=Perfil y ajustes… signed_in_as=Identificado como enable_javascript=Este sitio web funciona mejor con JavaScript. -toc=Tabla de contenido username=Nombre de usuario email=Correo electrónico From 5002ad6a97dc3f2058d1b3bb3360f520ed0f4c80 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 18 Apr 2020 17:18:06 +0100 Subject: [PATCH 07/14] as per @6543 Signed-off-by: Andrew Thornton --- modules/markup/markdown/renderconfig.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index 7f479d650d2c5..bef67e9e59bf2 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -65,9 +65,7 @@ func (rc *RenderConfig) ToRenderConfig(meta yaml.MapSlice) { rc.Meta = "none" case "table": rc.Meta = "table" - case "details": - rc.Meta = "details" - default: + default: // "details" rc.Meta = "details" } case yaml.MapSlice: @@ -88,9 +86,7 @@ func (rc *RenderConfig) ToRenderConfig(meta yaml.MapSlice) { rc.Meta = "none" case "table": rc.Meta = "table" - case "details": - rc.Meta = "details" - default: + default: // "details" rc.Meta = "details" } case "details_icon": From 1b1e7d9949cb577980dc01ad4311bfc1f085648e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 18 Apr 2020 17:18:32 +0100 Subject: [PATCH 08/14] make vendor Signed-off-by: Andrew Thornton --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1105ac57ae37c..0d575d492ee51 100644 --- a/go.mod +++ b/go.mod @@ -120,7 +120,7 @@ require ( gopkg.in/ini.v1 v1.52.0 gopkg.in/ldap.v3 v3.0.2 gopkg.in/testfixtures.v2 v2.5.0 - gopkg.in/yaml.v2 v2.2.4 + gopkg.in/yaml.v2 v2.2.8 mvdan.cc/xurls/v2 v2.1.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 xorm.io/builder v0.3.7 From 55c3e48638a75e5137dbc9021960c42e63efbd9c Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 18 Apr 2020 17:28:11 +0100 Subject: [PATCH 09/14] as per @guillep2k Signed-off-by: Andrew Thornton --- modules/markup/html.go | 48 ++++++++++++++---------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index a179385243316..7942c33b5e043 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -356,40 +356,24 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { if attr.Key != "class" { continue } - // Got a class - from := 0 - - for idx := strings.Index(attr.Val[from:], "icon") + from; idx >= from; { - if (idx != 0 && attr.Val[idx-1] != ' ') || - (len(attr.Val) > idx+4 && attr.Val[idx+4] != ' ') { - from = idx + 4 - continue + classes := strings.Split(attr.Val, " ") + for i, class := range classes { + if class == "icon" { + icon = true + classes[0], classes[i] = classes[i], classes[0] + attr.Val = strings.Join(classes, " ") + + // Remove all children of icons + child := node.FirstChild + for child != nil { + node.RemoveChild(child) + child = child.NextSibling + } + node.FirstChild = nil + node.LastChild = nil + break } - // We should be an icon - end := idx + 4 - - // now need to move the icon class first... - if idx != 0 && attr.Val[idx-1] == ' ' { - idx-- - } - class := "icon " + attr.Val[:idx] - if len(attr.Val) > end+1 && attr.Val[end] == ' ' { - class += " " + attr.Val[end:] - } - attr.Val = class - - // Remove all children of icons - child := node.FirstChild - for child != nil { - node.RemoveChild(child) - child = child.NextSibling - } - node.FirstChild = nil - node.LastChild = nil - - break } - break } } for n := node.FirstChild; n != nil; n = n.NextSibling { From 86775175d277937d6565887118967d724949f8a3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 18 Apr 2020 17:28:26 +0100 Subject: [PATCH 10/14] as per @guillep2k Signed-off-by: Andrew Thornton --- modules/markup/html.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index 7942c33b5e043..e294425b665e2 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -359,7 +359,6 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { classes := strings.Split(attr.Val, " ") for i, class := range classes { if class == "icon" { - icon = true classes[0], classes[i] = classes[i], classes[0] attr.Val = strings.Join(classes, " ") From 50601b2025ca69408000e7e730ab733dc760b80d Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 18 Apr 2020 19:19:27 +0100 Subject: [PATCH 11/14] Update modules/markup/html.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> --- modules/markup/html.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index e294425b665e2..4c70c64029be3 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -366,7 +366,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { child := node.FirstChild for child != nil { node.RemoveChild(child) - child = child.NextSibling + child = node.FirstChild } node.FirstChild = nil node.LastChild = nil From 7aaec85d73e33b437f267968580a4afb645405b4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 19 Apr 2020 14:15:34 +0100 Subject: [PATCH 12/14] as per @guillep2k Signed-off-by: Andrew Thornton --- modules/markup/html.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index 4c70c64029be3..294b870d8c6a4 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -368,8 +368,6 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { node.RemoveChild(child) child = node.FirstChild } - node.FirstChild = nil - node.LastChild = nil break } } From eda3f5fcb6ef72663d69620ab5829c25e2dd28d3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 23 Apr 2020 08:42:12 +0100 Subject: [PATCH 13/14] Fix level changes Signed-off-by: Andrew Thornton --- modules/markup/markdown/toc.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/markup/markdown/toc.go b/modules/markup/markdown/toc.go index d1476747e2559..189821c341abe 100644 --- a/modules/markup/markdown/toc.go +++ b/modules/markup/markdown/toc.go @@ -27,15 +27,15 @@ func createTOCNode(toc []Header, lang string) ast.Node { } } for _, header := range toc { - if header.Level < currentLevel { + for currentLevel > header.Level { ul = ul.Parent().(*ast.List) - } else if header.Level > currentLevel { - for currentLevel < header.Level { - newL := ast.NewList('-') - ul.AppendChild(ul, newL) - currentLevel++ - ul = newL - } + currentLevel-- + } + for currentLevel < header.Level { + newL := ast.NewList('-') + ul.AppendChild(ul, newL) + currentLevel++ + ul = newL } li := ast.NewListItem(currentLevel * 2) a := ast.NewLink() From 729570563fa04b1f66c77166530c0304d96f6c0b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Fri, 24 Apr 2020 12:54:32 +0100 Subject: [PATCH 14/14] Err... Drone do something!