anyproto/anytype-heart
1
0
Fork 0
mirror of https://github.com/anyproto/anytype-heart.git synced 2025-06-10 18:10:49 +09:00
Code Activity

[GO-678-dep-licence] replace lib for readability

Browse source
This commit is contained in:
Pavel Zavyalov 2023-01-10 12:56:53 +06:00
parent a818f104e5
commit 74f8e59f68
No known key found for this signature in database
GPG key ID: D12476F8A1107A26
3 changed files with 53 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

8
util/linkpreview/linkpreview.go
View file

@ -1,8 +1,10 @@
package linkpreview
import (
"bytes"
"context"
"github.com/anytypeio/go-anytype-middleware/util/text"
"github.com/go-shiori/go-readability"
"io"
"net/http"
"path/filepath"
@ -13,7 +15,6 @@ import (
"github.com/anytypeio/go-anytype-middleware/util/uri"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
"github.com/mauidude/go-readability"
"github.com/microcosm-cc/bluemonday"
"github.com/otiai10/opengraph/v2"
)
@ -101,11 +102,12 @@ func (l *linkPreview) findContent(data []byte) (content string) {
// ignore possible panic while html parsing
}
}()
doc, err := readability.NewDocument(string(data))
article, err := readability.FromReader(bytes.NewReader(data), nil)
if err != nil {
return
}
content = doc.Content()
content = article.TextContent
content = strings.TrimSpace(l.bmPolicy.Sanitize(content))
content = strings.Join(strings.Fields(content), " ") // removes repetitive whitespaces
if text.UTF16RuneCountString(content) > maxDescriptionSize {