diff --git a/core/block/bookmark/bookmark_service.go b/core/block/bookmark/bookmark_service.go index 891edc43f..33a3ad4cf 100644 --- a/core/block/bookmark/bookmark_service.go +++ b/core/block/bookmark/bookmark_service.go @@ -3,10 +3,6 @@ package bookmark import ( "context" "fmt" - "io" - "io/ioutil" - "net/http" - "os" "path/filepath" "strings" "sync" @@ -266,7 +262,7 @@ func (s *service) ContentUpdaters(spaceID string, url string, parseBlock bool) ( wg.Add(1) go func() { defer wg.Done() - hash, err := s.loadImage(spaceID, data.Title, data.ImageUrl) + hash, err := s.loadImage(spaceID, getFileNameFromURL(url, "cover"), data.ImageUrl) if err != nil { log.Errorf("load image: %s", err) return @@ -283,7 +279,7 @@ func (s *service) ContentUpdaters(spaceID string, url string, parseBlock bool) ( wg.Add(1) go func() { defer wg.Done() - hash, err := s.loadImage(spaceID, "", data.FaviconUrl) + hash, err := s.loadImage(spaceID, getFileNameFromURL(url, "icon"), data.FaviconUrl) if err != nil { log.Errorf("load favicon: %s", err) return @@ -358,55 +354,35 @@ func (s *service) fetcher(spaceID string, blockID string, params bookmark.FetchP return nil } +func getFileNameFromURL(url, filename string) string { + u, err := uri.ParseURI(url) + if err != nil { + return "" + } + if u.Hostname() == "" { + return "" + } + var urlFileExt string + lastPath := strings.Split(u.Path, "/") + if len(lastPath) > 0 { + urlFileExt = filepath.Ext(lastPath[len(lastPath)-1]) + } + + source := strings.TrimPrefix(u.Hostname(), "www.") + source = strings.ReplaceAll(source, ".", "_") + if source != "" { + source += "_" + } + source += filename + urlFileExt + return source +} + func (s *service) loadImage(spaceId string, title, url string) (hash string, err error) { uploader := s.fileUploaderFactory.NewUploader(spaceId, objectorigin.Bookmark()) - tempDir := s.tempDirService.TempDir() ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("download image: %s", resp.Status) - } - - tmpFile, err := ioutil.TempFile(tempDir, "anytype_downloaded_file_*") - if err != nil { - return "", err - } - defer os.Remove(tmpFile.Name()) - - _, err = io.Copy(tmpFile, resp.Body) - if err != nil { - return "", err - } - - _, err = tmpFile.Seek(0, io.SeekStart) - if err != nil { - return "", err - } - - fileName := strings.Split(filepath.Base(url), "?")[0] - if value := resp.Header.Get("Content-Disposition"); value != "" { - contentDisposition := strings.Split(value, "filename=") - if len(contentDisposition) > 1 { - fileName = strings.Trim(contentDisposition[1], "\"") - } - - } - - if title != "" { - fileName = title - } - res := uploader.SetName(fileName).SetFile(tmpFile.Name()).SetImageKind(model.ImageKind_AutomaticallyAdded).Upload(ctx) + res := uploader.SetName(title).SetUrl(url).SetImageKind(model.ImageKind_AutomaticallyAdded).Upload(ctx) return res.FileObjectId, res.Err } diff --git a/core/block/bookmark/bookmark_service_test.go b/core/block/bookmark/bookmark_service_test.go index 4ff969cdc..49f558574 100644 --- a/core/block/bookmark/bookmark_service_test.go +++ b/core/block/bookmark/bookmark_service_test.go @@ -159,6 +159,70 @@ func TestService_FetchBookmarkContent(t *testing.T) { }) } +func TestGetFileNameFromURL(t *testing.T) { + tests := []struct { + name string + url string + filename string + want string + }{ + { + name: "Valid URL with file extension, includes www", + url: "https://www.example.com/path/image.png", + filename: "myfile", + want: "example_com_myfile.png", + }, + { + name: "Valid URL without file extension", + url: "https://example.com/path/file", + filename: "myfile", + want: "example_com_myfile", + }, + { + name: "Trailing slash, no explicit file name in path", + url: "http://www.example.org/folder/", + filename: "test", + want: "example_org_test", + }, + { + name: "Invalid URL format", + url: "vvv", + filename: "file", + want: "", + }, + { + name: "Empty path (no trailing slash)", + url: "http://www.example.com", + filename: "file", + want: "example_com_file", + }, + { + name: "Complex domain without www", + url: "https://sub.example.co.uk/path/report.pdf", + filename: "report", + want: "sub_example_co_uk_report.pdf", + }, + { + name: "URL with query parameters (should ignore queries)", + url: "https://www.testsite.com/path/subpath/file.txt?key=value", + filename: "newfile", + want: "testsite_com_newfile.txt", + }, + { + name: "URL ends with a dot-based extension in path but no file name", + url: "https://www.example.net/path/.htaccess", + filename: "hidden", + want: "example_net_hidden.htaccess", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getFileNameFromURL(tt.url, tt.filename) + assert.Equal(t, tt.want, got, "Output filename should match the expected value") + }) + } +} + const testHtml = ` Title diff --git a/core/block/editor/archive.go b/core/block/editor/archive.go index 9af42990c..0f90d45f9 100644 --- a/core/block/editor/archive.go +++ b/core/block/editor/archive.go @@ -45,7 +45,6 @@ func (p *Archive) Init(ctx *smartblock.InitContext) (err error) { if err = p.SmartBlock.Init(ctx); err != nil { return } - p.DisableLayouts() p.AddHook(p.updateObjects, smartblock.HookAfterApply) return p.updateObjects(smartblock.ApplyInfo{}) diff --git a/core/block/editor/dashboard.go b/core/block/editor/dashboard.go index a972e641b..0dc58cad7 100644 --- a/core/block/editor/dashboard.go +++ b/core/block/editor/dashboard.go @@ -46,7 +46,6 @@ func (p *Dashboard) Init(ctx *smartblock.InitContext) (err error) { if err = p.SmartBlock.Init(ctx); err != nil { return } - p.DisableLayouts() p.AddHook(p.updateObjects, smartblock.HookAfterApply) return p.updateObjects(smartblock.ApplyInfo{}) diff --git a/core/block/editor/page.go b/core/block/editor/page.go index df88fa566..8f5b42a93 100644 --- a/core/block/editor/page.go +++ b/core/block/editor/page.go @@ -98,6 +98,7 @@ func (p *Page) Init(ctx *smartblock.InitContext) (err error) { migrateFilesToObjects(p, p.fileObjectService)(ctx.State) } + p.EnableLayouts() if p.isRelationDeleted(ctx) { // todo: move this to separate component go func() { diff --git a/core/block/editor/smartblock/smartblock.go b/core/block/editor/smartblock/smartblock.go index f67a9bc09..ad8abcfdd 100644 --- a/core/block/editor/smartblock/smartblock.go +++ b/core/block/editor/smartblock/smartblock.go @@ -163,7 +163,7 @@ type SmartBlock interface { SendEvent(msgs []*pb.EventMessage) ResetToVersion(s *state.State) (err error) - DisableLayouts() + EnableLayouts() EnabledRelationAsDependentObjects() AddHook(f HookCallback, events ...Hook) AddHookOnce(id string, f HookCallback, events ...Hook) @@ -236,7 +236,7 @@ type smartBlock struct { lastDepDetails map[string]*domain.Details restrictions restriction.Restrictions isDeleted bool - disableLayouts bool + enableLayouts bool includeRelationObjectsAsDependents bool // used by some clients @@ -616,8 +616,12 @@ func (sb *smartBlock) IsLocked() bool { return activeCount > 0 } -func (sb *smartBlock) DisableLayouts() { - sb.disableLayouts = true +func (sb *smartBlock) EnableLayouts() { + sb.enableLayouts = true +} + +func (sb *smartBlock) IsLayoutsEnabled() bool { + return sb.enableLayouts } func (sb *smartBlock) EnabledRelationAsDependentObjects() { @@ -704,7 +708,7 @@ func (sb *smartBlock) Apply(s *state.State, flags ...ApplyFlag) (err error) { migrationVersionUpdated = s.MigrationVersion() != parent.MigrationVersion() } - msgs, act, err := state.ApplyState(sb.SpaceID(), s, !sb.disableLayouts) + msgs, act, err := state.ApplyState(sb.SpaceID(), s, sb.enableLayouts) if err != nil { return } @@ -1058,7 +1062,7 @@ func (sb *smartBlock) StateAppend(f func(d state.Doc) (s *state.State, changes [ sb.updateRestrictions() sb.injectDerivedDetails(s, sb.SpaceID(), sb.Type()) sb.execHooks(HookBeforeApply, ApplyInfo{State: s}) - msgs, act, err := state.ApplyState(sb.SpaceID(), s, !sb.disableLayouts) + msgs, act, err := state.ApplyState(sb.SpaceID(), s, sb.enableLayouts) if err != nil { return err } @@ -1093,7 +1097,7 @@ func (sb *smartBlock) StateRebuild(d state.Doc) (err error) { d.(*state.State).SetParent(sb.Doc.(*state.State)) // todo: make store diff sb.execHooks(HookBeforeApply, ApplyInfo{State: d.(*state.State)}) - msgs, _, err := state.ApplyState(sb.SpaceID(), d.(*state.State), !sb.disableLayouts) + msgs, _, err := state.ApplyState(sb.SpaceID(), d.(*state.State), sb.enableLayouts) log.Infof("changes: stateRebuild: %d events", len(msgs)) if err != nil { // can't make diff - reopen doc diff --git a/core/block/editor/smartblock/smarttest/smarttest.go b/core/block/editor/smartblock/smarttest/smarttest.go index 82c543b35..45daaf46c 100644 --- a/core/block/editor/smartblock/smarttest/smarttest.go +++ b/core/block/editor/smartblock/smarttest/smarttest.go @@ -245,10 +245,14 @@ func (st *SmartTest) SetObjectTypes(objectTypes []domain.TypeKey) { st.Doc.(*state.State).SetObjectTypeKeys(objectTypes) } -func (st *SmartTest) DisableLayouts() { +func (st *SmartTest) EnableLayouts() { return } +func (st *SmartTest) IsLayoutsEnabled() bool { + return false +} + func (st *SmartTest) SendEvent(msgs []*pb.EventMessage) { return } diff --git a/core/block/editor/spaceview.go b/core/block/editor/spaceview.go index 1d7eaf657..787183472 100644 --- a/core/block/editor/spaceview.go +++ b/core/block/editor/spaceview.go @@ -78,7 +78,6 @@ func (s *SpaceView) Init(ctx *smartblock.InitContext) (err error) { } s.log = s.log.With("spaceId", spaceId) - s.DisableLayouts() info := spaceinfo.NewSpacePersistentInfoFromState(ctx.State) newInfo := spaceinfo.NewSpacePersistentInfo(spaceId) newInfo.SetAccountStatus(info.GetAccountStatus()). diff --git a/core/block/editor/state/state.go b/core/block/editor/state/state.go index 89c113aba..323bfb6bc 100644 --- a/core/block/editor/state/state.go +++ b/core/block/editor/state/state.go @@ -1127,7 +1127,7 @@ func (s *State) FileRelationKeys() []domain.RelationKey { } if rel.Key == bundle.RelationKeyCoverId.String() { coverType := s.Details().GetInt64(bundle.RelationKeyCoverType) - if (coverType == 1 || coverType == 4) && slice.FindPos(keys, domain.RelationKey(rel.Key)) == -1 { + if (coverType == 1 || coverType == 4 || coverType == 5) && slice.FindPos(keys, domain.RelationKey(rel.Key)) == -1 { keys = append(keys, domain.RelationKey(rel.Key)) } } diff --git a/core/block/editor/state/state_test.go b/core/block/editor/state/state_test.go index f1be9f452..170d97908 100644 --- a/core/block/editor/state/state_test.go +++ b/core/block/editor/state/state_test.go @@ -2943,6 +2943,23 @@ func TestState_FileRelationKeys(t *testing.T) { // then assert.Len(t, keys, 0) }) + t.Run("unsplash cover", func(t *testing.T) { + // given + s := &State{ + relationLinks: pbtypes.RelationLinks{ + {Key: bundle.RelationKeyCoverId.String()}, + }, + details: domain.NewDetailsFromMap(map[domain.RelationKey]domain.Value{ + bundle.RelationKeyCoverType: domain.Int64(5), + }), + } + + // when + keys := s.FileRelationKeys() + + // then + assert.Len(t, keys, 1) + }) } func TestState_AddRelationLinks(t *testing.T) { diff --git a/core/block/source/source.go b/core/block/source/source.go index 19025d14f..013fe37dd 100644 --- a/core/block/source/source.go +++ b/core/block/source/source.go @@ -335,6 +335,11 @@ func (s *source) buildState() (doc state.Doc, err error) { template.WithRelations([]domain.RelationKey{bundle.RelationKeyBacklinks})(st) } + if s.Type() == smartblock.SmartBlockTypeWidget { + // todo: remove this after 0.41 release + state.CleanupLayouts(st) + } + s.fileObjectMigrator.MigrateFiles(st, s.space, s.GetFileKeysSnapshot()) // Details in spaceview comes from Workspace object, so we don't need to migrate them if s.Type() != smartblock.SmartBlockTypeSpaceView { diff --git a/core/converter/md/md.go b/core/converter/md/md.go index 153359e94..71efd95e8 100644 --- a/core/converter/md/md.go +++ b/core/converter/md/md.go @@ -217,13 +217,17 @@ func (h *MD) renderFile(buf writer, in *renderState, b *model.Block) { if file == nil || file.State != model.BlockContentFile_Done { return } - name := escape.MarkdownCharacters(html.EscapeString(file.Name)) + title, filename, ok := h.getLinkInfo(file.TargetObjectId) + if !ok { + filename = h.fn.Get("files", file.TargetObjectId, filepath.Base(file.Name), filepath.Ext(file.Name)) + title = filepath.Base(file.Name) + } buf.WriteString(in.indent) if file.Type != model.BlockContentFile_Image { - fmt.Fprintf(buf, "[%s](%s) \n", name, h.fn.Get("files", file.TargetObjectId, filepath.Base(file.Name), filepath.Ext(file.Name))) + fmt.Fprintf(buf, "[%s](%s) \n", title, filename) h.fileHashes = append(h.fileHashes, file.TargetObjectId) } else { - fmt.Fprintf(buf, "![%s](%s) \n", name, h.fn.Get("files", file.TargetObjectId, filepath.Base(file.Name), filepath.Ext(file.Name))) + fmt.Fprintf(buf, "![%s](%s) \n", title, filename) h.imageHashes = append(h.imageHashes, file.TargetObjectId) } } @@ -394,12 +398,27 @@ func (h *MD) getLinkInfo(docId string) (title, filename string, ok bool) { return } title = info.GetString(bundle.RelationKeyName) + // if object is a file + layout := info.GetInt64(bundle.RelationKeyLayout) + if layout == int64(model.ObjectType_file) || layout == int64(model.ObjectType_image) || layout == int64(model.ObjectType_audio) || layout == int64(model.ObjectType_video) { + ext := info.GetString(bundle.RelationKeyFileExt) + if ext != "" { + ext = "." + ext + } + title = strings.TrimSuffix(title, ext) + if title == "" { + title = docId + } + filename = h.fn.Get("files", docId, title, ext) + return + } if title == "" { title = info.GetString(bundle.RelationKeySnippet) } if title == "" { title = docId } + filename = h.fn.Get("", docId, title, h.Ext()) return } diff --git a/core/files/fileuploader/uploader.go b/core/files/fileuploader/uploader.go index 0dc292746..229dc3cdc 100644 --- a/core/files/fileuploader/uploader.go +++ b/core/files/fileuploader/uploader.go @@ -272,7 +272,6 @@ func (u *uploader) SetUrl(url string) Uploader { if err != nil { // do nothing } - u.SetName(strings.Split(filepath.Base(url), "?")[0]) u.getReader = func(ctx context.Context) (*fileReader, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -287,6 +286,10 @@ func (u *uploader) SetUrl(url string) Uploader { return nil, err } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("failed to download url, status: %d", resp.StatusCode) + } + var fileName string if content := resp.Header.Get("Content-Disposition"); content != "" { contentDisposition := strings.Split(content, "filename=") @@ -294,6 +297,9 @@ func (u *uploader) SetUrl(url string) Uploader { fileName = strings.Trim(contentDisposition[1], "\"") } } + if fileName == "" { + fileName = uri.GetFileNameFromURLAndContentType(resp.Request.URL, resp.Header.Get("Content-Type")) + } tmpFile, err := ioutil.TempFile(u.tempDirProvider.TempDir(), "anytype_downloaded_file_*") if err != nil { @@ -332,7 +338,10 @@ func (u *uploader) SetUrl(url string) Uploader { } func (u *uploader) SetFile(path string) Uploader { - u.SetName(filepath.Base(path)) + if u.name == "" { + // only set name if it wasn't explicitly set before + u.SetName(filepath.Base(path)) + } u.setLastModifiedDate(path) u.getReader = func(ctx context.Context) (*fileReader, error) { @@ -420,7 +429,12 @@ func (u *uploader) Upload(ctx context.Context) (result UploadResult) { } if fileName := buf.GetFileName(); fileName != "" { - u.SetName(fileName) + if u.name == "" { + u.SetName(fileName) + } else if filepath.Ext(u.name) == "" { + // enrich current name with extension + u.name += filepath.Ext(fileName) + } } if u.block != nil { diff --git a/go.mod b/go.mod index 6af7b915e..a3c07420d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PuerkitoBio/goquery v1.10.1 github.com/VividCortex/ewma v1.2.0 github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 - github.com/anyproto/any-store v0.1.6 + github.com/anyproto/any-store v0.1.7 github.com/anyproto/any-sync v0.6.0-alpha.11.0.20250131145459-0bf102738a87 github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a github.com/anyproto/go-chash v0.1.0 @@ -31,7 +31,7 @@ require ( github.com/ethereum/go-ethereum v1.13.15 github.com/gin-gonic/gin v1.10.0 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 - github.com/go-chi/chi/v5 v5.2.0 + github.com/go-chi/chi/v5 v5.2.1 github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f github.com/goccy/go-graphviz v0.2.9 github.com/gogo/protobuf v1.3.2 @@ -104,19 +104,18 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 - golang.org/x/image v0.23.0 + golang.org/x/image v0.24.0 golang.org/x/mobile v0.0.0-20241108191957-fa514ef75a0f golang.org/x/net v0.34.0 golang.org/x/oauth2 v0.25.0 golang.org/x/sys v0.29.0 - golang.org/x/text v0.21.0 + golang.org/x/text v0.22.0 google.golang.org/grpc v1.70.0 gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20180125164251-1832d8546a9f gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 storj.io/drpc v0.0.34 zombiezen.com/go/sqlite v1.4.0 - ) require ( @@ -284,7 +283,7 @@ require ( golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.32.0 // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/sync v0.10.0 // indirect + golang.org/x/sync v0.11.0 // indirect golang.org/x/term v0.28.0 // indirect golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.29.0 // indirect diff --git a/go.sum b/go.sum index 4d931e0ca..e7f1f9f70 100644 --- a/go.sum +++ b/go.sum @@ -82,10 +82,10 @@ github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= -github.com/anyproto/any-store v0.1.6 h1:CmxIH2JhgHH2s7dnv1SNKAQ3spvX2amG/5RbrTZkLaQ= -github.com/anyproto/any-store v0.1.6/go.mod h1:nbyRoJYOlvSWU1xDOrmgPP96UeoTf4eYZ9k+qqLK9k8= github.com/anyproto/any-sync v0.6.0-alpha.11.0.20250131145459-0bf102738a87 h1:Vkkz4b2e/ARu2NO2UhP6OxBUtrfBslVvtF9uCcYraRg= github.com/anyproto/any-sync v0.6.0-alpha.11.0.20250131145459-0bf102738a87/go.mod h1:JmjcChDSqgt6+G697YI3Yr57SZswxPJNX1arN//UYig= +github.com/anyproto/any-store v0.1.7 h1:E3DntI+JXo3h7v0WTUJWH+nm7G4MV0PNOXZ6SFzQ2OU= +github.com/anyproto/any-store v0.1.7/go.mod h1:nbyRoJYOlvSWU1xDOrmgPP96UeoTf4eYZ9k+qqLK9k8= github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a h1:ZZM+0OUCQMWSLSflpkf0ZMVo3V76qEDDIXPpQOClNs0= github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a/go.mod h1:4fkueCZcGniSMXkrwESO8zzERrh/L7WHimRNWecfGM0= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc= @@ -323,8 +323,8 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0= -github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8= +github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.0.2/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= @@ -1214,8 +1214,8 @@ golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86h golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68= -golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY= +golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ= +golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1338,8 +1338,9 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1453,8 +1454,9 @@ golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/lib/bundle/relation.gen.go b/pkg/lib/bundle/relation.gen.go index 3e27bd3bf..39b4d575d 100644 --- a/pkg/lib/bundle/relation.gen.go +++ b/pkg/lib/bundle/relation.gen.go @@ -378,7 +378,7 @@ var ( RelationKeyCoverType: { DataSource: model.Relation_details, - Description: "1-image, 2-color, 3-gradient, 4-prebuilt bg image. Value stored in coverId", + Description: "1-image, 2-color, 3-gradient, 4-prebuilt bg image, 5 - unsplash image. Value stored in coverId", Format: model.RelationFormat_number, Hidden: true, Id: "_brcoverType", diff --git a/space/techspace/mock_techspace/mock_AccountObject.go b/space/techspace/mock_techspace/mock_AccountObject.go index a43daba2c..1003d71d0 100644 --- a/space/techspace/mock_techspace/mock_AccountObject.go +++ b/space/techspace/mock_techspace/mock_AccountObject.go @@ -574,34 +574,34 @@ func (_c *MockAccountObject_Details_Call) RunAndReturn(run func() *domain.Detail return _c } -// DisableLayouts provides a mock function with given fields: -func (_m *MockAccountObject) DisableLayouts() { +// EnableLayouts provides a mock function with given fields: +func (_m *MockAccountObject) EnableLayouts() { _m.Called() } -// MockAccountObject_DisableLayouts_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DisableLayouts' -type MockAccountObject_DisableLayouts_Call struct { +// MockAccountObject_EnableLayouts_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnableLayouts' +type MockAccountObject_EnableLayouts_Call struct { *mock.Call } -// DisableLayouts is a helper method to define mock.On call -func (_e *MockAccountObject_Expecter) DisableLayouts() *MockAccountObject_DisableLayouts_Call { - return &MockAccountObject_DisableLayouts_Call{Call: _e.mock.On("DisableLayouts")} +// EnableLayouts is a helper method to define mock.On call +func (_e *MockAccountObject_Expecter) EnableLayouts() *MockAccountObject_EnableLayouts_Call { + return &MockAccountObject_EnableLayouts_Call{Call: _e.mock.On("EnableLayouts")} } -func (_c *MockAccountObject_DisableLayouts_Call) Run(run func()) *MockAccountObject_DisableLayouts_Call { +func (_c *MockAccountObject_EnableLayouts_Call) Run(run func()) *MockAccountObject_EnableLayouts_Call { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } -func (_c *MockAccountObject_DisableLayouts_Call) Return() *MockAccountObject_DisableLayouts_Call { +func (_c *MockAccountObject_EnableLayouts_Call) Return() *MockAccountObject_EnableLayouts_Call { _c.Call.Return() return _c } -func (_c *MockAccountObject_DisableLayouts_Call) RunAndReturn(run func()) *MockAccountObject_DisableLayouts_Call { +func (_c *MockAccountObject_EnableLayouts_Call) RunAndReturn(run func()) *MockAccountObject_EnableLayouts_Call { _c.Call.Return(run) return _c } diff --git a/util/uri/uri.go b/util/uri/uri.go index ca80ceae5..a8336750b 100644 --- a/util/uri/uri.go +++ b/util/uri/uri.go @@ -2,8 +2,10 @@ package uri import ( "fmt" + "mime" "net/url" "os" + "path/filepath" "regexp" "strings" ) @@ -84,3 +86,68 @@ func NormalizeAndParseURI(uri string) (*url.URL, error) { return url.Parse(normalizeURI(uri)) } + +var preferredExtensions = map[string]string{ + "image/jpeg": ".jpeg", + "audio/mpeg": ".mp3", + // Add more preferred mappings if needed +} + +func GetFileNameFromURLAndContentType(u *url.URL, contentType string) string { + var host string + if u != nil { + + lastSegment := filepath.Base(u.Path) + // Determine if this looks like a real filename. We'll say it's real if it has a dot or is a hidden file starting with a dot. + if lastSegment == "." || lastSegment == "" || (!strings.HasPrefix(lastSegment, ".") && !strings.Contains(lastSegment, ".")) { + // Not a valid filename + lastSegment = "" + } + + if lastSegment != "" { + // A plausible filename was found directly in the URL + return lastSegment + } + + // No filename, fallback to host-based + host = strings.TrimPrefix(u.Hostname(), "www.") + host = strings.ReplaceAll(host, ".", "_") + if host == "" { + host = "file" + } + } + + // Try to get a preferred extension for the content type + var ext string + if preferred, ok := preferredExtensions[contentType]; ok { + ext = preferred + } else { + extensions, err := mime.ExtensionsByType(contentType) + if err != nil || len(extensions) == 0 { + // Fallback if no known extension + extensions = []string{".bin"} + } + ext = extensions[0] + } + + // Determine a base name from content type + base := "file" + if strings.HasPrefix(contentType, "image/") { + base = "image" + } else if strings.HasPrefix(contentType, "audio/") { + base = "audio" + } else if strings.HasPrefix(contentType, "video/") { + base = "video" + } + + var res strings.Builder + if host != "" { + res.WriteString(host) + res.WriteString("_") + } + res.WriteString(base) + if ext != "" { + res.WriteString(ext) + } + return res.String() +} diff --git a/util/uri/uri_test.go b/util/uri/uri_test.go index 29fe0ad06..208b3fe52 100644 --- a/util/uri/uri_test.go +++ b/util/uri/uri_test.go @@ -1,6 +1,7 @@ package uri import ( + "net/url" "testing" "github.com/stretchr/testify/assert" @@ -118,3 +119,114 @@ func TestURI_ValidateURI(t *testing.T) { assert.NoError(t, err) }) } + +func TestGetFileNameFromURLWithContentTypeAndMime(t *testing.T) { + mustParseURL := func(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + t.Fatalf("url.Parse(%q) failed: %v", s, err) + } + return u + } + + tests := []struct { + name string + url *url.URL + contentType string + expected string + }{ + { + name: "URL with explicit filename and extension", + url: mustParseURL("https://example.com/image.jpg"), + contentType: "image/jpeg", + expected: "image.jpg", + }, + { + name: "URL with explicit filename and extension, but wrong content type", + url: mustParseURL("https://example.com/image.jpg"), + contentType: "image/png", + expected: "image.jpg", + }, + { + name: "URL with explicit filename and extension, and empty content type", + url: mustParseURL("https://example.com/image.jpg"), + contentType: "", + expected: "image.jpg", + }, + { + name: "URL with query and fragment, explicit filename", + url: mustParseURL("https://example.com/file.jpeg?query=1#111"), + contentType: "image/jpeg", + expected: "file.jpeg", + }, + { + name: "No filename in URL, fallback to host and image/jpeg", + url: mustParseURL("https://www.example.com/path/to/"), + contentType: "image/jpeg", + // host -> example_com + // image/jpeg typically corresponds to .jpeg or .jpg (mime usually returns .jpeg) + expected: "example_com_image.jpeg", + }, + { + name: "Host-only URL, fallback with image/png", + url: mustParseURL("https://www.example.com"), + contentType: "image/png", + expected: "example_com_image.png", + }, + { + name: "Filename present with video/mp4", + url: mustParseURL("https://www.sub.example.co.uk/folder/video.mp4"), + contentType: "video/mp4", + expected: "video.mp4", + }, + { + name: "No extension but filename present", + url: mustParseURL("https://example.com/filename"), + contentType: "image/gif", + expected: "example_com_image.gif", + }, + { + name: "Invalid URL returns empty", + url: nil, + contentType: "image/jpeg", + expected: "image.jpeg", + }, + { + name: "No filename, video/unknown fallback to .bin", + url: mustParseURL("https://www.subdomain.example.com/folder/"), + contentType: "video/unknown", + // no known extension for "video/unknown", fallback .bin + expected: "subdomain_example_com_video.bin", + }, + { + name: "Hidden file as filename", + url: mustParseURL("https://example.com/.htaccess"), + contentType: "text/plain", + expected: ".htaccess", + }, + { + name: "URL with query but no filename extension, fallback audio/mpeg", + url: mustParseURL("https://example.com/path?version=2"), + contentType: "audio/mpeg", + // audio/mpeg known extension: .mp3 + expected: "example_com_audio.mp3", + }, + { + name: "Unknown type entirely", + url: mustParseURL("https://example.net/"), + contentType: "application/x-something-strange", + // no filename, fallback host: example_net + // unknown type -> .bin + expected: "example_net_file.bin", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetFileNameFromURLAndContentType(tt.url, tt.contentType) + if got != tt.expected { + t.Errorf("GetFileNameFromURL(%q, %q) = %q; want %q", tt.url, tt.contentType, got, tt.expected) + } + }) + } +}