anyproto/anytype-heart
1
0
Fork 0
mirror of https://github.com/anyproto/anytype-heart.git synced 2025-06-09 17:44:59 +09:00
Code Activity

GO-677 Unify URL validation

Browse source
This commit is contained in:
kirillston 2022-12-21 11:11:10 +01:00
parent 4c0c4d8d37
commit 5e67fdc4d1
No known key found for this signature in database
GPG key ID: 88218A7F1109754B
12 changed files with 89 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

2
core/block/editor/bookmark/bookmark.go
View file

@ -61,7 +61,7 @@ func (b *sbookmark) fetch(s *state.State, id, url string, isSync bool) (err erro
if b == nil {
return smartblock.ErrSimpleBlockNotFound
}
url, err = uri.ProcessURI(url)
url, err = uri.URIManager.ValidateAndNormalizeURI(url)
if err != nil {
// Do nothing
}

2
core/block/editor/file/uploader.go
View file

@ -175,7 +175,7 @@ func (u *uploader) AddOptions(options ...files.AddOption) Uploader {
}
func (u *uploader) SetUrl(url string) Uploader {
url, _ = uri.ProcessURI(url)
url = uri.URIManager.NormalizeURI(url)
u.name = strings.Split(filepath.Base(url), "?")[0]
u.getReader = func(ctx context.Context) (*fileReader, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)

3
core/block/editor/import/import.go
View file

@ -27,6 +27,7 @@ import (
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/anytypeio/go-anytype-middleware/util/slice"
"github.com/anytypeio/go-anytype-middleware/util/uri"
)
var (
@ -669,7 +670,7 @@ func (imp *importImpl) convertTextToPageLink(block *model.Block) {
}
func (imp *importImpl) convertTextToBookmark(block *model.Block) {
if _, err := url.Parse(block.GetText().Marks.Marks[0].Param); err != nil {
if err := uri.URIManager.Validate(block.GetText().Marks.Marks[0].Param); err != nil {
return
}

4
core/block/import/markdown/blockconverter.go
View file

@ -6,7 +6,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"strings"
@ -18,6 +17,7 @@ import (
"github.com/anytypeio/go-anytype-middleware/core/block/import/markdown/anymark"
"github.com/anytypeio/go-anytype-middleware/pb"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
"github.com/anytypeio/go-anytype-middleware/util/uri"
)
type Service interface {
@ -258,7 +258,7 @@ func (m *mdConverter) convertTextToPageLink(block *model.Block) {
}
func (m *mdConverter) convertTextToBookmark(block *model.Block) {
if _, err := url.Parse(block.GetText().Marks.Marks[0].Param); err != nil {
if err := uri.URIManager.Validate(block.GetText().Marks.Marks[0].Param); err != nil {
return
}

4
core/block/service.go
View file

@ -1593,7 +1593,7 @@ func (s *Service) fetchBookmarkContent(url string) bookmarksvc.ContentFuture {
func (s *Service) ObjectCreateBookmark(
req pb.RpcObjectCreateBookmarkRequest,
) (objectId string, newDetails *types.Struct, err error) {
u, err := uri.ProcessURI(pbtypes.GetString(req.Details, bundle.RelationKeySource.String()))
u, err := uri.URIManager.ValidateAndNormalizeURI(pbtypes.GetString(req.Details, bundle.RelationKeySource.String()))
if err != nil {
return "", nil, fmt.Errorf("process uri: %w", err)
}
@ -1602,7 +1602,7 @@ func (s *Service) ObjectCreateBookmark(
}
func (s *Service) ObjectBookmarkFetch(req pb.RpcObjectBookmarkFetchRequest) (err error) {
url, err := uri.ProcessURI(req.Url)
url, err := uri.URIManager.ValidateAndNormalizeURI(req.Url)
if err != nil {
return fmt.Errorf("process uri: %w", err)
}

10
core/block/simple/text/text.go
View file

@ -234,7 +234,7 @@ func (t *Text) SetText(text string, marks *model.BlockContentTextMarks) (err err
} else {
for mI, _ := range marks.Marks {
if marks.Marks[mI].Type == model.BlockContentTextMark_Link {
m, err := uri.ProcessURI(marks.Marks[mI].Param)
m, err := uri.URIManager.ValidateAndNormalizeURI(marks.Marks[mI].Param)
if err == nil {
marks.Marks[mI].Param = m
}
@ -700,10 +700,10 @@ func (t *Text) IsEmpty() bool {
return false
}
func isIncompatibleType(firstType, secondType model.BlockContentTextMarkType ) bool {
if (firstType == model.BlockContentTextMark_Link && secondType == model.BlockContentTextMark_Object) ||
(secondType == model.BlockContentTextMark_Link && firstType == model.BlockContentTextMark_Object) {
func isIncompatibleType(firstType, secondType model.BlockContentTextMarkType) bool {
if (firstType == model.BlockContentTextMark_Link && secondType == model.BlockContentTextMark_Object) ||
(secondType == model.BlockContentTextMark_Link && firstType == model.BlockContentTextMark_Object) {
return true
}
return false
}
}

6
core/converter/md/md.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"html"
"io"
"net/url"
"path/filepath"
"sort"
"strings"
@ -21,6 +20,7 @@ import (
"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/anytypeio/go-anytype-middleware/util/uri"
)
type FileNamer interface {
@ -217,7 +217,7 @@ func (h *MD) renderBookmark(buf writer, in *renderState, b *model.Block) {
bm := b.GetBookmark()
if bm != nil && bm.Url != "" {
buf.WriteString(in.indent)
url, e := url.Parse(bm.Url)
url, e := uri.URIManager.ValidateAndParseURI(bm.Url)
if e == nil {
fmt.Fprintf(buf, "[%s](%s) \n", escape.MarkdownCharacters(html.EscapeString(bm.Title)), url.String())
}
@ -411,7 +411,7 @@ func (mw *marksWriter) writeMarks(buf writer, pos int) {
if start {
buf.WriteString("[")
} else {
urlP, e := url.Parse(m.Param)
urlP, e := uri.URIManager.ValidateAndParseURI(m.Param)
urlS := m.Param
if e == nil {
urlS = urlP.String()

17
core/linkpreview.go
View file

@ -2,7 +2,7 @@ package core
import (
"context"
"net/url"
"fmt"
"strings"
"time"
@ -15,25 +15,16 @@ func (mw *Middleware) LinkPreview(cctx context.Context, req *pb.RpcLinkPreviewRe
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
urlStr, err := uri.ProcessURI(req.Url)
urlStr, err := uri.URIManager.ValidateAndNormalizeURI(req.Url)
if err != nil {
return &pb.RpcLinkPreviewResponse{
Error: &pb.RpcLinkPreviewResponseError{
Code: pb.RpcLinkPreviewResponseError_UNKNOWN_ERROR,
Description: err.Error(),
},
}
}
u, err := url.Parse(urlStr)
if err != nil {
return &pb.RpcLinkPreviewResponse{
Error: &pb.RpcLinkPreviewResponseError{
Code: pb.RpcLinkPreviewResponseError_UNKNOWN_ERROR,
Description: "failed to parse url",
Description: fmt.Sprintf("failed to parse url: %v", err),
},
}
}
u := uri.URIManager.ParseURI(urlStr)
mw.m.RLock()
defer mw.m.RUnlock()

4
core/relation/service.go
View file

@ -3,7 +3,6 @@ package relation
import (
"errors"
"fmt"
"net/url"
"strings"
"sync"
@ -20,6 +19,7 @@ import (
"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/anytypeio/go-anytype-middleware/util/uri"
)
const CName = "relation"
@ -379,7 +379,7 @@ func (s *service) ValidateFormat(key string, v *types.Value) error {
return fmt.Errorf("incorrect type: %T instead of string", v.Kind)
}
_, err := url.Parse(strings.TrimSpace(v.GetStringValue()))
err := uri.URIManager.Validate(strings.TrimSpace(v.GetStringValue()))
if err != nil {
return fmt.Errorf("failed to parse URL: %s", err.Error())
}

5
util/linkpreview/linkpreview.go
View file

@ -5,7 +5,6 @@ import (
"github.com/anytypeio/go-anytype-middleware/util/text"
"io"
"net/http"
"net/url"
"path/filepath"
"strings"
"unicode/utf8"
@ -87,7 +86,7 @@ func (l *linkPreview) convertOGToInfo(fetchUrl string, og *opengraph.OpenGraph)
}
if len(og.Image) != 0 {
url, err := uri.ProcessURI(og.Image[0].URL)
url, err := uri.URIManager.ValidateAndNormalizeURI(og.Image[0].URL)
if err == nil {
i.ImageUrl = url
}
@ -127,7 +126,7 @@ func (l *linkPreview) makeNonHtml(fetchUrl string, resp *http.Response) (i model
} else {
i.Type = model.LinkPreview_Unknown
}
pUrl, e := url.Parse(fetchUrl)
pUrl, e := uri.URIManager.ValidateAndParseURI(fetchUrl)
if e == nil {
pUrl.Path = "favicon.ico"
pUrl.RawQuery = ""

4
util/unsplash/unsplash.go
View file

@ -8,6 +8,7 @@ import (
"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
"github.com/anytypeio/go-anytype-middleware/util/ocache"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/anytypeio/go-anytype-middleware/util/uri"
"github.com/dsoprea/go-exif/v3"
jpegstructure "github.com/dsoprea/go-jpeg-image-structure/v2"
"github.com/hbagdi/go-unsplash/unsplash"
@ -15,7 +16,6 @@ import (
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
@ -111,7 +111,7 @@ func newFromPhoto(v unsplash.Photo) (Result, error) {
fUrl := v.Urls.Regular.String()
// hack to have full hd instead of 1080w,
// in case unsplash will change the URL format it will not break things
u, _ := url.Parse(fUrl)
u := uri.URIManager.ParseURI(fUrl)
if u != nil {
if q := u.Query(); q.Get("w") != "" {
q.Set("w", "1920")

114
util/uri/uri.go
View file

@ -2,11 +2,10 @@ package uri
import (
"fmt"
"net/url"
"os"
"regexp"
"strings"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
)
var (
@ -19,63 +18,76 @@ var (
winFilepathPrefixRegex = regexp.MustCompile(`^[a-zA-Z]:[\\\/]`)
)
func ValidateEmail(email string) bool {
if len(email) == 0 {
return false
}
return noPrefixEmailRegexp.MatchString(email)
type Validator interface {
Validate(string) error
}
func ValidatePhone(phone string) bool {
if len(phone) == 0 {
return false
}
return noPrefixTelRegexp.MatchString(phone)
type Parser interface {
ParseURI(string) *url.URL
}
// ProcessURI tries to verify the web URI and return the normalized URI
func ProcessURI(url string) (urlOut string, err error) {
if len(url) == 0 {
return url, fmt.Errorf("url is empty")
} else if noPrefixEmailRegexp.MatchString(url) {
return "mailto:" + url, nil
} else if noPrefixTelRegexp.MatchString(url) {
return "tel:" + url, nil
} else if winFilepathPrefixRegex.MatchString(url) {
return "", fmt.Errorf("filepath not supported")
} else if strings.HasPrefix(url, string(os.PathSeparator)) || strings.HasPrefix(url, ".") {
return "", fmt.Errorf("filepath not supported")
} else if noPrefixHttpRegex.MatchString(url) {
return "http://" + url, nil
} else if haveUriSchemeRegex.MatchString(url) {
return url, nil
}
return url, fmt.Errorf("not a uri")
type Normalizer interface {
NormalizeURI(string) string
}
func ProcessAllURI(blocks []*model.Block) []*model.Block {
for bI, _ := range blocks {
if blocks[bI].GetText() != nil && blocks[bI].GetText().Marks != nil && len(blocks[bI].GetText().Marks.Marks) > 0 {
marks := blocks[bI].GetText().Marks.Marks
type Manager struct {
Validator
Parser
Normalizer
}
for mI, _ := range marks {
if marks[mI].Type == model.BlockContentTextMark_Link {
marks[mI].Param, _ = ProcessURI(marks[mI].Param)
}
}
var URIManager Manager
blocks[bI].GetText().Marks.Marks = marks
}
func (m Manager) Validate(uri string) error {
uri = strings.TrimSpace(uri)
if len(uri) == 0 {
return fmt.Errorf("url is empty")
} else if winFilepathPrefixRegex.MatchString(uri) {
return fmt.Errorf("filepath not supported")
} else if strings.HasPrefix(uri, string(os.PathSeparator)) || strings.HasPrefix(uri, ".") {
return fmt.Errorf("filepath not supported")
}
return blocks
_, err := url.Parse(uri)
return err
}
func (m Manager) ParseURI(uri string) *url.URL {
u, _ := url.Parse(uri)
return u
}
func (m Manager) NormalizeURI(uri string) string {
if noPrefixEmailRegexp.MatchString(uri) {
return "mailto:" + uri
} else if noPrefixTelRegexp.MatchString(uri) {
return "tel:" + uri
} else if noPrefixHttpRegex.MatchString(uri) {
return "http://" + uri
}
return uri
}
func (m Manager) ValidateAndParseURI(uri string) (*url.URL, error) {
uri = strings.TrimSpace(uri)
if len(uri) == 0 {
return nil, fmt.Errorf("url is empty")
} else if winFilepathPrefixRegex.MatchString(uri) {
return nil, fmt.Errorf("filepath not supported")
} else if strings.HasPrefix(uri, string(os.PathSeparator)) || strings.HasPrefix(uri, ".") {
return nil, fmt.Errorf("filepath not supported")
}
_, err := url.Parse(uri)
return nil, err
}
func (m Manager) ValidateAndNormalizeURI(uri string) (string, error) {
err := m.Validate(uri)
if err != nil {
return "", err
}
return m.NormalizeURI(uri), nil
}