1
0
Fork 0
mirror of https://github.com/anyproto/anytype-heart.git synced 2025-06-10 01:51:07 +09:00

Merge branch 'master' of github.com:anytypeio/go-anytype-middleware into feat-subscriptions-dataview-rels

This commit is contained in:
Sergey Cherepanov 2022-03-21 14:37:43 +03:00
commit e7270ce2d9
No known key found for this signature in database
GPG key ID: 87F8EDE8FBDF637C
13 changed files with 426 additions and 154 deletions

View file

@ -20,14 +20,13 @@ setup: setup-go
@echo 'Setting up npm...'
@npm install
uuu:
@echo $(PATH)
setup-go:
@echo 'Setting up go modules...'
@go mod download
@GO111MODULE=off go get github.com/ahmetb/govvv
@GO111MODULE=off go get golang.org/x/mobile/cmd/...
go install golang.org/x/mobile/cmd/gomobile@latest
go install golang.org/x/mobile/cmd/gobind@latest
fmt:
@echo 'Formatting with prettier...'
@ -84,16 +83,22 @@ build-js-addon:
@rm clientlibrary/jsaddon/lib.a clientlibrary/jsaddon/lib.h clientlibrary/jsaddon/bridge.h
build-ios: setup-go
gomobile init
@go get golang.org/x/mobile/bind
@echo 'Building library for iOS...'
@$(eval FLAGS := $$(shell govvv -flags | sed 's/main/github.com\/anytypeio\/go-anytype-middleware\/core/g'))
@GOPRIVATE=github.com/anytypeio gomobile bind -tags "nogrpcserver gomobile" -ldflags "$(FLAGS)" -v -target=ios -o Lib.xcframework github.com/anytypeio/go-anytype-middleware/clientlibrary/service github.com/anytypeio/go-anytype-middleware/core
gomobile bind -tags "nogrpcserver gomobile" -ldflags "$(FLAGS)" -v -target=ios -o Lib.xcframework github.com/anytypeio/go-anytype-middleware/clientlibrary/service github.com/anytypeio/go-anytype-middleware/core
@mkdir -p dist/ios/ && mv Lib.xcframework dist/ios/
@go mod tidy
build-android: setup-go
gomobile init
@go get golang.org/x/mobile/bind
@echo 'Building library for Android...'
@$(eval FLAGS := $$(shell govvv -flags | sed 's/main/github.com\/anytypeio\/go-anytype-middleware\/core/g'))
@GOPRIVATE=github.com/anytypeio gomobile bind -tags "nogrpcserver gomobile" -ldflags "$(FLAGS)" -v -target=android -o lib.aar github.com/anytypeio/go-anytype-middleware/clientlibrary/service github.com/anytypeio/go-anytype-middleware/core
gomobile bind -tags "nogrpcserver gomobile" -ldflags "$(FLAGS)" -v -target=android -o lib.aar github.com/anytypeio/go-anytype-middleware/clientlibrary/service github.com/anytypeio/go-anytype-middleware/core
@mkdir -p dist/android/ && mv lib.aar dist/android/
@go mod tidy
setup-protoc-go:
@echo 'Setting up protobuf compiler...'
@ -108,8 +113,9 @@ setup-protoc-go:
setup-protoc-jsweb:
@echo 'Installing grpc-web plugin...'
@rm -rf grpc-web
@git clone https://github.com/grpc/grpc-web
@$(MAKE) -C grpc-web install-plugin
@git clone http://github.com/grpc/grpc-web
git apply ./clientlibrary/jsaddon/grpcweb_mac.patch
@[ -d "/opt/homebrew" ] && PREFIX="/opt/homebrew/bin" $(MAKE) -C grpc-web install-plugin || $(MAKE) -C grpc-web install-plugin
@rm -rf grpc-web
setup-protoc-doc:

View file

@ -0,0 +1,20 @@
Index: grpc-web/javascript/net/grpc/web/generator/Makefile
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/grpc-web/javascript/net/grpc/web/generator/Makefile b/grpc-web/javascript/net/grpc/web/generator/Makefile
--- a/grpc-web/javascript/net/grpc/web/generator/Makefile
+++ b/grpc-web/javascript/net/grpc/web/generator/Makefile
@@ -13,9 +13,9 @@
# limitations under the License.
CXX ?= g++
-CPPFLAGS += -I/usr/local/include -pthread
+CPPFLAGS += -I/usr/local/include -I/opt/homebrew/include -pthread
CXXFLAGS += -std=c++11
-LDFLAGS += -L/usr/local/lib -lprotoc -lprotobuf -lpthread -ldl
+LDFLAGS += -L/usr/local/lib -L/opt/homebrew/lib -lprotoc -lprotobuf -lpthread -ldl
PREFIX ?= /usr/local
MIN_MACOS_VERSION := 10.7 # Supports OS X Lion
STATIC ?= yes

View file

@ -1145,7 +1145,7 @@ func (mw *Middleware) DownloadFile(req *pb.RpcDownloadFileRequest) *pb.RpcDownlo
}
if req.Path == "" {
req.Path = os.TempDir() + string(os.PathSeparator) + "anytype-download"
req.Path = mw.GetAnytype().TempDir() + string(os.PathSeparator) + "anytype-download"
}
err := os.MkdirAll(req.Path, 0755)

View file

@ -88,7 +88,7 @@ func (mw *Middleware) GetAnytype() core.Service {
mw.m.RLock()
defer mw.m.RUnlock()
if mw.app != nil {
return mw.app.MustComponent(core.CName).(core.Service)
return mw.app.MustComponent("anytype").(core.Service)
}
return nil
}

5
go.mod
View file

@ -13,6 +13,7 @@ require (
github.com/blevesearch/bleve/v2 v2.3.0
github.com/cheggaaa/mb v1.0.3
github.com/dave/jennifer v1.4.1
github.com/davecgh/go-spew v1.1.1
github.com/dgraph-io/badger v1.6.2
github.com/dgtony/collections v0.1.6
github.com/dhowden/tag v0.0.0-20201120070457-d52dcb253c63
@ -82,7 +83,7 @@ require (
github.com/tyler-smith/go-bip39 v1.0.1-0.20190808214741-c55f737395bc
github.com/uber/jaeger-client-go v2.28.0+incompatible
github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
github.com/yuin/goldmark v1.3.5
github.com/yuin/goldmark v1.4.0
go.uber.org/zap v1.16.0
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
@ -90,7 +91,7 @@ require (
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 // indirect
golang.org/x/text v0.3.7
golang.org/x/tools v0.1.5 // indirect
golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 // indirect
google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0 // indirect
google.golang.org/grpc v1.40.0
gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20180125164251-1832d8546a9f

9
go.sum
View file

@ -1492,8 +1492,9 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0 h1:OtISOGfH6sOWa1/qXqqAiOIAO6Z5J3AEAE18WAq6BiQ=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
@ -1669,6 +1670,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -1772,6 +1774,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU=
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@ -1849,8 +1852,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 h1:YuekqPskqwCCPM79F1X5Dhv4ezTCj+Ki1oNwiafxkA0=
golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View file

@ -3,6 +3,9 @@ package mill
import (
"bytes"
"fmt"
"github.com/disintegration/imaging"
"github.com/dsoprea/go-exif/v3"
jpegstructure "github.com/dsoprea/go-jpeg-image-structure/v2"
"image"
"image/color/palette"
"image/draw"
@ -12,9 +15,6 @@ import (
"io"
"strconv"
"github.com/disintegration/imaging"
"github.com/rwcarlsen/goexif/exif"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/mill/ico"
)
@ -72,7 +72,7 @@ func (m *ImageResize) Options(add map[string]interface{}) (string, error) {
}
func (m *ImageResize) Mill(r io.ReadSeeker, name string) (*Result, error) {
img, formatStr, err := image.Decode(r)
imgConfig, formatStr, err := image.DecodeConfig(r)
if err != nil {
return nil, err
}
@ -83,11 +83,7 @@ func (m *ImageResize) Mill(r io.ReadSeeker, name string) (*Result, error) {
return nil, err
}
clean, err := removeExif(r, img, format)
if err != nil {
return nil, err
}
var height int
width, err := strconv.Atoi(m.Opts.Width)
if err != nil {
return nil, fmt.Errorf("invalid width: " + m.Opts.Width)
@ -97,161 +93,188 @@ func (m *ImageResize) Mill(r io.ReadSeeker, name string) (*Result, error) {
return nil, fmt.Errorf("invalid quality: " + m.Opts.Quality)
}
buff, rect, err := encodeImage(clean, format, width, quality)
if err != nil {
return nil, err
}
var (
img image.Image
orientation int
)
return &Result{
File: buff,
Meta: map[string]interface{}{
"width": rect.Dx(),
"height": rect.Dy(),
},
}, nil
}
// removeExif strips exif data from an image
func removeExif(reader io.Reader, img image.Image, format Format) (io.Reader, error) {
if format == GIF || format == ICO {
return reader, nil
}
exf, _ := exif.Decode(reader)
var err error
img, err = correctOrientation(img, exf)
if err != nil {
return nil, err
}
// re-encoding will remove any exif
return encodeSingleImage(img, format)
}
// encodeImage creates a jpeg|gif from reader (quality applies to jpeg only)
// NOTE: format is the reader image format, destination format is chosen accordingly.
func encodeImage(reader io.Reader, format Format, width int, quality int) (*bytes.Buffer, *image.Rectangle, error) {
buff := new(bytes.Buffer)
var size image.Rectangle
if format != GIF {
// encode to png or jpeg
img, _, err := image.Decode(reader)
if format == JPEG {
var exifData []byte
exifData, err = getExifData(r)
if err != nil {
return nil, nil, err
return nil, fmt.Errorf("failed to get exif data %s", err.Error())
}
if img.Bounds().Size().X < width {
width = img.Bounds().Size().X
_, err = r.Seek(0, io.SeekStart)
if err != nil {
return nil, err
}
if exifData != nil {
orientation, err = getJpegOrientation(exifData)
if err != nil {
return nil, fmt.Errorf("failed to get jpeg orientation: %s", err.Error())
}
_, err = r.Seek(0, io.SeekStart)
if err != nil {
return nil, err
}
}
if orientation > 1 {
img, err = jpeg.Decode(r)
if err != nil {
return nil, err
}
img = reverseOrientation(img, orientation)
if err != nil {
err = fmt.Errorf("failed to fix img orientation: %s", err.Error())
return nil, err
}
imgConfig.Width, imgConfig.Height = img.Bounds().Max.X, img.Bounds().Max.Y
}
}
if imgConfig.Width <= width || width == 0 {
// we will not do the upscale
width, height = imgConfig.Width, imgConfig.Height
}
if orientation <= 1 && width == imgConfig.Width {
var r2 io.Reader
if format == JPEG {
r2, err = patchReaderRemoveExif(r)
if err != nil {
return nil, err
}
} else {
r2 = r
}
// here is an optimisation
// lets return the original picture in case it has not been resized or normilised
return &Result{
File: r2,
Meta: map[string]interface{}{
"width": imgConfig.Width,
"height": imgConfig.Height,
},
}, nil
}
if format == JPEG || format == PNG || format == ICO {
if format == JPEG && img == nil {
// we already have img decoded if we have orientation <= 1
img, err = jpeg.Decode(r)
if err != nil {
return nil, err
}
} else if format != JPEG {
img, err = png.Decode(r)
if err != nil {
return nil, err
}
}
resized := imaging.Resize(img, width, 0, imaging.Lanczos)
width, height = resized.Rect.Max.X, resized.Rect.Max.Y
if format == PNG || format == ICO {
if err = png.Encode(buff, resized); err != nil {
return nil, nil, err
buff := &bytes.Buffer{}
if format == JPEG {
if err = jpeg.Encode(buff, resized, &jpeg.Options{Quality: quality}); err != nil {
return nil, err
}
} else {
if err = jpeg.Encode(buff, resized, &jpeg.Options{Quality: quality}); err != nil {
return nil, nil, err
if err = png.Encode(buff, resized); err != nil {
return nil, err
}
}
size = resized.Rect
} else {
// encode to gif
img, err := gif.DecodeAll(reader)
return &Result{
File: buff,
Meta: map[string]interface{}{
"width": width,
"height": height,
},
}, nil
} else if format == GIF {
gifImg, err := gif.DecodeAll(r)
if err != nil {
return nil, nil, err
return nil, err
}
if len(img.Image) == 0 {
return nil, nil, fmt.Errorf("gif does not have any frames")
}
firstFrame := img.Image[0].Bounds()
if firstFrame.Dx() < width {
width = firstFrame.Dx()
}
rect := image.Rect(0, 0, firstFrame.Dx(), firstFrame.Dy())
rect := image.Rect(0, 0, imgConfig.Width, imgConfig.Height)
rgba := image.NewRGBA(rect)
for index, frame := range img.Image {
for index, frame := range gifImg.Image {
bounds := frame.Bounds()
draw.Draw(rgba, bounds, frame, bounds.Min, draw.Over)
img.Image[index] = imageToPaletted(imaging.Resize(rgba, width, 0, imaging.Lanczos))
gifImg.Image[index] = imageToPaletted(imaging.Resize(rgba, width, 0, imaging.Lanczos))
}
gifImg.Config.Width, gifImg.Config.Height = gifImg.Image[0].Bounds().Dx(), gifImg.Image[0].Bounds().Dy()
buff := bytes.NewBuffer(make([]byte, 0))
if err = gif.EncodeAll(buff, gifImg); err != nil {
return nil, err
}
img.Config.Width = img.Image[0].Bounds().Dx()
img.Config.Height = img.Image[0].Bounds().Dy()
if err = gif.EncodeAll(buff, img); err != nil {
return nil, nil, err
}
size = img.Image[0].Bounds()
return &Result{
File: buff,
Meta: map[string]interface{}{
"width": gifImg.Config.Width,
"height": gifImg.Config.Height,
},
}, nil
}
return buff, &size, nil
return nil, fmt.Errorf("unknown format")
}
// correctOrientation returns a copy of an image (jpg|png|gif) with exif removed
func correctOrientation(img image.Image, exf *exif.Exif) (image.Image, error) {
if exf == nil {
return img, nil
}
orient, err := exf.Get(exif.Orientation)
if err != nil && err != exif.TagNotPresentError(exif.Orientation) {
return nil, err
}
if orient != nil {
img = reverseOrientation(img, orient.String())
} else {
img = reverseOrientation(img, "1")
}
return img, nil
}
// encodeSingleImage creates a reader from an image
func encodeSingleImage(img image.Image, format Format) (*bytes.Reader, error) {
writer := &bytes.Buffer{}
var err error
switch format {
case JPEG:
err = jpeg.Encode(writer, img, &jpeg.Options{Quality: 100})
case PNG:
// NOTE: while PNGs don't technically have exif data,
// they can contain meta data with sensitive info
err = png.Encode(writer, img)
default:
err = fmt.Errorf("unrecognized image format")
}
func getExifData(r io.ReadSeeker) (data []byte, err error) {
exifData, err := exif.SearchAndExtractExifWithReader(r)
if err != nil {
if err == exif.ErrNoExif {
return nil, nil
}
return nil, err
}
return bytes.NewReader(writer.Bytes()), nil
return exifData, nil
}
func getJpegOrientation(exifData []byte) (int, error) {
tags, _, err := exif.GetFlatExifData(exifData, nil)
if err != nil {
return 0, err
}
var orientation int
for _, tag := range tags {
if tag.TagId != 274 {
continue
}
if v, ok := tag.Value.([]uint16); ok && len(v) == 1 {
orientation = int(v[0])
}
}
return orientation, nil
}
// reverseOrientation transforms the given orientation to 1
func reverseOrientation(img image.Image, orientation string) *image.NRGBA {
func reverseOrientation(img image.Image, orientation int) image.Image {
switch orientation {
case "1":
case 1:
return imaging.Clone(img)
case "2":
case 2:
return imaging.FlipV(img)
case "3":
case 3:
return imaging.Rotate180(img)
case "4":
case 4:
return imaging.Rotate180(imaging.FlipV(img))
case "5":
case 5:
return imaging.Rotate270(imaging.FlipV(img))
case "6":
case 6:
return imaging.Rotate270(img)
case "7":
case 7:
return imaging.Rotate90(imaging.FlipV(img))
case "8":
case 8:
return imaging.Rotate90(img)
}
@ -266,3 +289,31 @@ func imageToPaletted(img image.Image) *image.Paletted {
draw.FloydSteinberg.Draw(pm, b, img, image.ZP)
return pm
}
func patchReaderRemoveExif(r io.ReadSeeker) (io.Reader, error) {
jmp := jpegstructure.NewJpegMediaParser()
size, err := r.Seek(0, io.SeekEnd)
if err != nil {
return nil, err
}
_, _ = r.Seek(0, io.SeekStart)
buff := bytes.NewBuffer(make([]byte, 0, size))
intfc, err := jmp.Parse(r, int(size))
if err != nil {
return nil, fmt.Errorf("failed to open file to read exif: %s", err.Error())
}
sl := intfc.(*jpegstructure.SegmentList)
_, err = sl.DropExif()
if err != nil {
return nil, err
}
err = sl.Write(buff)
if err != nil {
return nil, err
}
return buff, nil
}

View file

@ -1,8 +1,15 @@
package mill
import (
"bytes"
"fmt"
"github.com/davecgh/go-spew/spew"
exif2 "github.com/dsoprea/go-exif/v3"
"github.com/stretchr/testify/require"
"image"
"image/jpeg"
"io"
"io/ioutil"
"os"
"testing"
@ -13,6 +20,126 @@ import (
var errFailedToFindExifMarker = fmt.Errorf("exif: failed to find exif intro marker")
func TestImageResize_Mill_ShouldRotateAndRemoveExif(t *testing.T) {
configs := []*ImageResize{
{
Opts: ImageResizeOpts{
Width: "0",
Quality: "100",
},
},
{
Opts: ImageResizeOpts{
Width: "2200",
Quality: "100",
},
},
{
Opts: ImageResizeOpts{
Width: "1800",
Quality: "100",
},
},
}
for _, cfg := range configs {
file, err := os.Open(testdata.Images[0].Path)
if err != nil {
t.Fatal(err)
}
imgCfg, err := jpeg.DecodeConfig(file)
// the picture is rotated 90 degrees
require.Equal(t, 1200, imgCfg.Width)
require.Equal(t, 1800, imgCfg.Height)
file.Seek(0, io.SeekStart)
res, err := cfg.Mill(file, "test")
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadAll(res.File)
if err != nil {
t.Fatal(err)
}
imgCfg, err = jpeg.DecodeConfig(bytes.NewReader(b))
require.NoError(t, err)
require.Equal(t, 1800, imgCfg.Width)
require.Equal(t, 1200, imgCfg.Height)
d, err := exif2.SearchAndExtractExif(b)
require.Error(t, exif2.ErrNoExif, err)
require.Nil(t, d)
}
}
func TestImageResize_Mill_ShouldNotBeReencoded(t *testing.T) {
configs := []*ImageResize{
{
Opts: ImageResizeOpts{
Width: "0",
Quality: "100",
},
},
{
Opts: ImageResizeOpts{
Width: "680", // same
Quality: "80",
},
},
{
Opts: ImageResizeOpts{
Width: "1000", // larger
Quality: "100",
},
},
}
file, err := os.Open(testdata.Images[1].Path)
if err != nil {
t.Fatal(err)
}
origImg, err := jpeg.Decode(file)
origImgDump := spew.Sdump(*(origImg.(*image.YCbCr)))
for _, cfg := range configs {
file, err := os.Open(testdata.Images[1].Path)
if err != nil {
t.Fatal(err)
}
imgCfg, err := jpeg.DecodeConfig(file)
// the picture is rotated 90 degrees
require.Equal(t, 680, imgCfg.Width)
require.Equal(t, 518, imgCfg.Height)
file.Seek(0, io.SeekStart)
res, err := cfg.Mill(file, "test")
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadAll(res.File)
if err != nil {
t.Fatal(err)
}
img, err := jpeg.Decode(bytes.NewReader(b))
require.NoError(t, err)
require.Equal(t, 680, img.Bounds().Max.X)
require.Equal(t, 518, img.Bounds().Max.Y)
d, err := exif2.SearchAndExtractExif(b)
require.Error(t, exif2.ErrNoExif, err)
require.Nil(t, d)
require.Equal(t, origImgDump, spew.Sdump(*(img.(*image.YCbCr))))
}
}
func TestImageResize_Mill(t *testing.T) {
m := &ImageResize{
Opts: ImageResizeOpts{
@ -44,3 +171,21 @@ func TestImageResize_Mill(t *testing.T) {
file.Close()
}
}
func Test_patchReaderRemoveExif(t *testing.T) {
f, err := os.Open(testdata.Images[0].Path)
s, _ := f.Stat()
fmt.Println(s.Size())
require.NoError(t, err)
_, err = getExifData(f)
require.NoError(t, err)
f.Seek(0, io.SeekStart)
clean, err := patchReaderRemoveExif(f)
require.NoError(t, err)
b, err := ioutil.ReadAll(clean)
require.NoError(t, err)
_, _, err = image.Decode(bytes.NewReader(b))
require.NoError(t, err)
}

View file

@ -9,7 +9,11 @@ var Image = `
"use": ":file",
"pin": true,
"plaintext": false,
"mill": "/blob"
"mill": "/image/resize",
"opts": {
"width": "0",
"quality": "100"
}
},
"large": {
"use": ":file",

BIN
pkg/lib/mill/testdata/Landscape_8.jpg vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 344 KiB

View file

@ -9,6 +9,20 @@ type TestImage struct {
}
var Images = []TestImage{
{
Path: "testdata/Landscape_8.jpg",
Format: "jpeg",
HasExif: true,
Width: 1200,
Height: 1800,
},
{
Path: "testdata/image-no-orientation.jpg",
Format: "jpeg",
HasExif: true,
Width: 680,
Height: 518,
},
{
Path: "testdata/image.jpeg",
Format: "jpeg",

View file

@ -27,8 +27,8 @@ func WriteReaderIntoFileReuseSameExistingFile(path string, r io.Reader) (string,
}
var (
ext = filepath.Ext(path)
dir = filepath.Dir(path)
ext = filepath.Ext(path)
dir = filepath.Dir(path)
name = strings.TrimSuffix(filepath.Base(path), ext)
)

View file

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/anytypeio/go-anytype-middleware/app"
"github.com/anytypeio/go-anytype-middleware/core/configfetcher"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
"github.com/anytypeio/go-anytype-middleware/util/ocache"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/dsoprea/go-exif/v3"
@ -14,6 +15,7 @@ import (
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
@ -22,6 +24,8 @@ import (
"time"
)
var log = logging.Logger("unsplash")
const (
CName = "unsplash"
DefaultToken = "TLKq5P192MptAcTHnGM8WQPZV8kKNn1eT9FEi5Srem0"
@ -36,17 +40,23 @@ type Unsplash interface {
app.Component
}
type tempDirGetter interface {
TempDir() string
}
type unsplashService struct {
mu sync.Mutex
cache ocache.OCache
client *unsplash.Unsplash
limit int
config configfetcher.ConfigFetcher
mu sync.Mutex
cache ocache.OCache
client *unsplash.Unsplash
limit int
config configfetcher.ConfigFetcher
tempDirGetter tempDirGetter
}
func (l *unsplashService) Init(app *app.App) (err error) {
l.cache = ocache.New(l.search, ocache.WithTTL(cacheTTL), ocache.WithGCPeriod(cacheGCPeriod))
l.config = app.MustComponent(configfetcher.CName).(configfetcher.ConfigFetcher)
l.tempDirGetter = app.MustComponent("anytype").(tempDirGetter)
return
}
@ -68,6 +78,7 @@ type Result struct {
PictureThumbUrl string
PictureSmallUrl string
PictureFullUrl string
PictureHDUrl string
Artist string
ArtistURL string
}
@ -96,6 +107,19 @@ func newFromPhoto(v unsplash.Photo) (Result, error) {
if v.Urls.Small != nil {
res.PictureSmallUrl = v.Urls.Small.String()
}
if v.Urls.Regular != nil {
fUrl := v.Urls.Regular.String()
// hack to have full hd instead of 1080w,
// in case unsplash will change the URL format it will not break things
u, _ := url.Parse(fUrl)
if u != nil {
if q := u.Query(); q.Get("w") != "" {
q.Set("w", "1920")
u.RawQuery = q.Encode()
}
}
res.PictureHDUrl = u.String()
}
if v.Urls.Full != nil {
res.PictureFullUrl = v.Urls.Full.String()
}
@ -204,16 +228,11 @@ func (l *unsplashService) Download(ctx context.Context, id string) (imgPath stri
}
}
// we must call download endpoint according to the API guidelines
picUrl, _, err := l.client.Photos.DownloadLink(id)
req, err := http.NewRequest("GET", picture.PictureHDUrl, nil)
if err != nil {
return "", err
}
req, err := http.NewRequest("GET", picUrl.String(), nil)
if err != nil {
return "", err
}
req = req.WithContext(ctx)
client := http.DefaultClient
resp, err := client.Do(req)
@ -221,7 +240,7 @@ func (l *unsplashService) Download(ctx context.Context, id string) (imgPath stri
return "", fmt.Errorf("failed to download file from unsplash: %s", err.Error())
}
defer resp.Body.Close()
tmpfile, err := ioutil.TempFile(os.TempDir(), picture.ID)
tmpfile, err := ioutil.TempFile(l.tempDirGetter.TempDir(), picture.ID)
if err != nil {
return "", fmt.Errorf("failed to create temp file: %s", err.Error())
}
@ -236,6 +255,15 @@ func (l *unsplashService) Download(ctx context.Context, id string) (imgPath stri
if err != nil {
return "", fmt.Errorf("failed to inject exif: %s", err.Error())
}
go func(cl *unsplash.Unsplash) {
// we must call download endpoint according to the API guidelines
// but we can do it in a separate goroutine to make sure we will download the picture as fast as possible
_, _, err = cl.Photos.DownloadLink(id)
if err != nil {
log.Errorf("failed to call unsplash download endpoint: %s", err.Error())
}
}(l.client)
return p, nil
}