pomp/fusion
1
0
Fork 0
forked from 0x2E/fusion
Code Activity

feat(#5): create groups and import feeds into them, based on opml

Browse source
This commit is contained in:
rook1e 2024-08-25 18:40:34 +08:00
parent 4f7db53bce
commit 8beb858c40
No known key found for this signature in database
GPG key ID: C63289D731719BC0
12 changed files with 178 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

5
api/group.go
View file

@ -33,11 +33,12 @@ func (f groupAPI) Create(c echo.Context) error {
return err
}
if err := f.srv.Create(c.Request().Context(), &req); err != nil {
resp, err := f.srv.Create(c.Request().Context(), &req)
if err != nil {
return err
}
return c.NoContent(http.StatusCreated)
return c.JSON(http.StatusCreated, resp)
}
func (f groupAPI) Update(c echo.Context) error {

12
frontend/src/lib/api/group.ts
View file

@ -7,11 +7,13 @@ export async function allGroups() {
}
export async function createGroup(name: string) {
return await api.post('groups', {
json: {
name: name
}
});
return await api
.post('groups', {
json: {
name: name
}
})
.json<{ id: number }>();
}
export async function updateGroup(id: number, name: string) {

59
frontend/src/lib/opml.ts
View file

@ -1,18 +1,55 @@
export function parse(content: string) {
const feeds: { name: string; link: string }[] = [];
const xmlDoc = new DOMParser().parseFromString(content, 'text/xml');
const outlines = xmlDoc.getElementsByTagName('outline');
type feedT = {
name: string;
link: string;
};
type groupT = {
name: string;
feeds: feedT[];
};
const groups = new Map<string, groupT>();
const defaultGroup = { name: 'Default', feeds: [] };
groups.set('Default', defaultGroup);
for (let i = 0; i < outlines.length; i++) {
const outline = outlines.item(i);
if (!outline) continue;
const link = outline.getAttribute('xmlUrl') || outline.getAttribute('htmlUrl') || '';
if (!link) continue;
const name = outline.getAttribute('title') || outline.getAttribute('text') || '';
feeds.push({ name, link });
function dfs(parentGroup: groupT | null, node: Element) {
if (node.tagName !== 'outline') {
return;
}
if (node.getAttribute('type')?.toLowerCase() == 'rss') {
if (!parentGroup) {
parentGroup = defaultGroup;
}
parentGroup.feeds.push({
name: node.getAttribute('title') || node.getAttribute('text') || '',
link: node.getAttribute('xmlUrl') || node.getAttribute('htmlUrl') || ''
});
return;
}
if (!node.children.length) {
return;
}
const nodeName = node.getAttribute('text') || node.getAttribute('title') || '';
const name = parentGroup ? parentGroup.name + '/' + nodeName : nodeName;
let curGroup = groups.get(name);
if (!curGroup) {
curGroup = { name: name, feeds: [] };
groups.set(name, curGroup);
}
for (const n of node.children) {
dfs(curGroup, n);
}
}
return feeds;
const xmlDoc = new DOMParser().parseFromString(content, 'text/xml');
const body = xmlDoc.getElementsByTagName('body')[0];
if (!body) {
return [];
}
for (const n of body.children) {
dfs(null, n);
}
return Array.from(groups.values());
}
export function dump(data: { name: string; feeds: { name: string; link: string }[] }[]) {

8
frontend/src/routes/(authed)/feeds/ActionAdd.svelte
View file

@ -106,6 +106,9 @@
}}
required
/>
<p class="text-sm text-muted-foreground">
The existing feed with the same link will be override.
</p>
</div>
<div>
@ -121,11 +124,6 @@
}}
required
/>
{#if formData.name}
<p class="text-sm text-muted-foreground">
The existing feed with the same link will be renamed as <b>{formData.name}</b>.
</p>
{/if}
</div>
<div>

95
frontend/src/routes/(authed)/feeds/ActionOPML.svelte
View file

@ -1,7 +1,6 @@
<script lang="ts">
import type { groupFeeds } from './+page';
import * as Sheet from '$lib/components/ui/sheet';
import * as Select from '$lib/components/ui/select';
import * as Tabs from '$lib/components/ui/tabs';
import { Button } from '$lib/components/ui/button';
import { Label } from '$lib/components/ui/label';
@ -9,22 +8,26 @@
import { toast } from 'svelte-sonner';
import { invalidateAll } from '$app/navigation';
import { dump, parse } from '$lib/opml';
import { FolderIcon } from 'lucide-svelte';
import { createGroup } from '$lib/api/group';
export let groups: groupFeeds[];
export let open: boolean;
let uploadedOpmls: FileList;
$: parseOPML(uploadedOpmls);
let opmlGroup = { id: groups[0].id, name: groups[0].name };
let parsedOpmlFeeds: { name: string; link: string }[] = [];
let parsedGroupFeeds: { name: string; feeds: { name: string; link: string }[] }[] = [];
let importing = false;
$: {
if (!open) {
parsedOpmlFeeds = [];
parsedGroupFeeds = [];
}
}
function parseOPML(opmls: FileList) {
if (!opmls) return;
const reader = new FileReader();
reader.onload = (f) => {
const content = f.target?.result?.toString();
@ -32,19 +35,37 @@
toast.error('Failed to load file content');
return;
}
parsedOpmlFeeds = parse(content);
console.log(parsedOpmlFeeds);
parsedGroupFeeds = parse(content).filter((v) => v.feeds.length > 0);
console.log(parsedGroupFeeds);
};
reader.readAsText(opmls[0]);
}
async function handleImportFeeds() {
try {
await createFeed({ group_id: opmlGroup.id, feeds: parsedOpmlFeeds });
toast.success('Feeds have been imported. Refreshing is running in the background');
} catch (e) {
toast.error((e as Error).message);
importing = true;
let success = 0;
const existingGroups = groups.map((v) => {
return { id: v.id, name: v.name };
});
for (const g of parsedGroupFeeds) {
try {
let groupID = existingGroups.find((v) => v.name === g.name)?.id;
if (groupID === undefined) {
groupID = (await createGroup(g.name)).id;
toast.success(`Created group ${g.name}`);
}
await createFeed({ group_id: groupID, feeds: g.feeds });
toast.success(`Imported into group ${g.name}`);
success++;
} catch (e) {
toast.error(`Failed to import group ${g.name}, error: ${(e as Error).message}`);
break;
}
}
if (success === parsedGroupFeeds.length) {
toast.success('All feeds have been imported. Refreshing is running in the background');
}
importing = false;
invalidateAll();
}
@ -69,7 +90,7 @@
</script>
<Sheet.Root bind:open>
<Sheet.Content class="w-full md:w-auto">
<Sheet.Content class="w-full md:max-w-[700px] overflow-y-auto">
<Sheet.Header>
<Sheet.Title>Import or Export Feeds</Sheet.Title>
<Sheet.Description>
@ -87,25 +108,6 @@
</Tabs.List>
<Tabs.Content value="import">
<form class="space-y-2" on:submit|preventDefault={handleImportFeeds}>
<div>
<Label for="group">Group</Label>
<Select.Root
disabled={groups.length < 2}
items={groups.map((v) => {
return { value: v.id, label: v.name };
})}
onSelectedChange={(v) => v && (opmlGroup.id = v.value)}
>
<Select.Trigger>
<Select.Value placeholder={opmlGroup.name} />
</Select.Trigger>
<Select.Content>
{#each groups as g}
<Select.Item value={g.id}>{g.name}</Select.Item>
{/each}
</Select.Content>
</Select.Root>
</div>
<div>
<Label for="feed_file">File</Label>
<input
@ -117,21 +119,34 @@
class="flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50"
/>
</div>
{#if parsedOpmlFeeds.length > 0}
{#if parsedGroupFeeds.length > 0}
<div>
<p class="text-sm text-muted-foreground">Parsed out {parsedOpmlFeeds.length} feeds</p>
<p class="text-sm text-green-700">Parsed successfully.</p>
<div
class="max-h-[200px] overflow-scroll p-2 rounded-md border bg-muted text-muted-foreground text-nowrap"
class="p-2 rounded-md border bg-muted/40 text-muted-foreground text-nowrap overflow-x-auto"
>
<ul>
{#each parsedOpmlFeeds as feed, index}
<li>{index + 1}. <b>{feed.name}</b> {feed.link}</li>
{/each}
</ul>
{#each parsedGroupFeeds as group}
<div class="flex flex-row items-center gap-1">
<FolderIcon size={14} />{group.name}
</div>
<ul class="list-inside list-decimal ml-[2ch] [&:not(:last-child)]:mb-2">
{#each group.feeds as feed}
<li>{feed.name}, {feed.link}</li>
{/each}
</ul>
{/each}
</div>
</div>
{/if}
<Button type="submit">Import</Button>
<div class="text-sm text-secondary-foreground">
<p>Note:</p>
<p>
1. Feeds will be imported into the corresponding group, which will be created
automatically if it does not exist.
</p>
<p>2. The existing feed with the same link will be override.</p>
</div>
<Button type="submit" disabled={importing}>Import</Button>
</form>
</Tabs.Content>
<Tabs.Content value="export">

2
frontend/src/routes/+layout.svelte
View file

@ -11,7 +11,7 @@
</svelte:head>
<ModeWatcher defaultMode="system" />
<Toaster position="top-right" richColors closeButton />
<Toaster position="top-right" richColors closeButton visibleToasts={10} />
<!-- h-screen does not work properly on mobile. Use calc(100dvh) instead.
https://stackoverflow.com/a/76120728/12812480 -->

4
model/feed.go
View file

@ -16,10 +16,10 @@ type Feed struct {
ID uint `gorm:"primarykey"`
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt soft_delete.DeletedAt
DeletedAt soft_delete.DeletedAt `gorm:"uniqueIndex:idx_link"`
Name *string `gorm:"name;not null"`
Link *string `gorm:"link;not null"` // FIX: unique index?
Link *string `gorm:"link;not null;uniqueIndex:idx_link"`
// LastBuild is the last time the content of the feed changed
LastBuild *time.Time `gorm:"last_build"`
// Failure is the reason of failure. If it is not null or empty, the fetch processor

6
repo/feed.go
View file

@ -6,6 +6,7 @@ import (
"github.com/0x2e/fusion/model"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
func NewFeed(db *gorm.DB) *Feed {
@ -47,7 +48,10 @@ func (f Feed) Get(id uint) (*model.Feed, error) {
}
func (f Feed) Create(data []*model.Feed) error {
return f.db.Create(data).Error
return f.db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "link"}, {Name: "deleted_at"}},
DoUpdates: clause.AssignmentColumns([]string{"name", "link"}),
}).Create(data).Error
}
func (f Feed) Update(id uint, feed *model.Feed) error {

43
repo/repo.go
View file

@ -2,6 +2,7 @@ package repo
import (
"errors"
"log"
"github.com/0x2e/fusion/conf"
"github.com/0x2e/fusion/model"
@ -27,6 +28,48 @@ func Init() {
}
func migrage() {
// The verison after v0.8.7 will add a unique index to Feed.Link.
// We must delete any duplicate feeds before AutoMigrate applies the
// new unique constraint.
err := DB.Transaction(func(tx *gorm.DB) error {
// query duplicate feeds
dupFeeds := make([]model.Feed, 0)
err := tx.Model(&model.Feed{}).Where(
"link IN (?)",
tx.Model(&model.Feed{}).Select("link").Group("link").
Having("count(link) > 1"),
).Order("link, id").Find(&dupFeeds).Error
if err != nil {
return err
}
// filter out feeds that will be deleted.
// we've queried with order, so the first one is the one we should keep.
distinct := map[string]uint{}
deleteIDs := make([]uint, 0, len(dupFeeds))
for _, f := range dupFeeds {
if _, ok := distinct[*f.Link]; !ok {
distinct[*f.Link] = f.ID
continue
}
deleteIDs = append(deleteIDs, f.ID)
log.Println("delete duplicate feed: ", f.ID, *f.Name, *f.Link)
}
if len(deleteIDs) > 0 {
// **hard** delete duplicate feeds and their items
err = tx.Where("id IN ?", deleteIDs).Unscoped().Delete(&model.Feed{}).Error
if err != nil {
return err
}
return tx.Where("feed_id IN ?", deleteIDs).Unscoped().Delete(&model.Item{}).Error
}
return nil
})
if err != nil {
panic(err)
}
// FIX: gorm not auto drop index and change 'not null'
if err := DB.AutoMigrate(&model.Feed{}, &model.Group{}, &model.Item{}); err != nil {
panic(err)

3
server/feed.go
View file

@ -93,9 +93,6 @@ func (f Feed) Create(ctx context.Context, req *ReqFeedCreate) error {
}
if err := f.repo.Create(feeds); err != nil {
if errors.Is(err, repo.ErrDuplicatedKey) {
err = NewBizError(err, http.StatusBadRequest, "link is not allowed to be the same as other feeds")
}
return err
}

11
server/group.go
View file

@ -46,15 +46,18 @@ func (g Group) All(ctx context.Context) (*RespGroupAll, error) {
}, nil
}
func (g Group) Create(ctx context.Context, req *ReqGroupCreate) error {
func (g Group) Create(ctx context.Context, req *ReqGroupCreate) (*RespGroupCreate, error) {
newGroup := &model.Group{
Name: req.Name,
}
err := g.repo.Create(newGroup)
if errors.Is(err, repo.ErrDuplicatedKey) {
err = NewBizError(err, http.StatusBadRequest, "name is not allowed to be the same as other groups")
if err != nil {
if errors.Is(err, repo.ErrDuplicatedKey) {
err = NewBizError(err, http.StatusBadRequest, "name is not allowed to be the same as other groups")
}
return nil, err
}
return err
return &RespGroupCreate{ID: newGroup.ID}, nil
}
func (g Group) Update(ctx context.Context, req *ReqGroupUpdate) error {

4
server/group_form.go
View file

@ -13,6 +13,10 @@ type ReqGroupCreate struct {
Name *string `json:"name" validate:"required"`
}
type RespGroupCreate struct {
ID uint `json:"id"`
}
type ReqGroupUpdate struct {
ID uint `param:"id" validate:"required"`
Name *string `json:"name" validate:"required"`