make common module as any-sync repo

commonfile/fileblockstore/fileblockstore.go

@@ -0,0 +1,50 @@
+package fileblockstore
+
+import (
+	"context"
+	"github.com/anytypeio/any-sync/app/logger"
+	"github.com/anytypeio/any-sync/commonfile/fileproto/fileprotoerr"
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+)
+
+var log = logger.NewNamed(CName)
+
+var (
+	ErrCIDNotFound   = fileprotoerr.ErrCIDNotFound
+	ErrCIDUnexpected = fileprotoerr.ErrUnexpected
+)
+
+const CName = "common.commonfile.fileblockstore"
+
+type ctxKey uint
+
+const (
+	ctxKeySpaceId ctxKey = iota
+)
+
+type BlockStore interface {
+	Get(ctx context.Context, k cid.Cid) (blocks.Block, error)
+	GetMany(ctx context.Context, ks []cid.Cid) <-chan blocks.Block
+	Add(ctx context.Context, b []blocks.Block) error
+	Delete(ctx context.Context, c cid.Cid) error
+}
+
+type BlockStoreLocal interface {
+	BlockStore
+	ExistsCids(ctx context.Context, ks []cid.Cid) (exists []cid.Cid, err error)
+	NotExistsBlocks(ctx context.Context, bs []blocks.Block) (notExists []blocks.Block, err error)
+}
+
+type BlockStoreSpaceIds interface {
+	SpaceIds() []string
+}
+
+func CtxWithSpaceId(ctx context.Context, spaceId string) context.Context {
+	return context.WithValue(ctx, ctxKeySpaceId, spaceId)
+}
+
+func CtxGetSpaceId(ctx context.Context) (spaceId string) {
+	spaceId, _ = ctx.Value(ctxKeySpaceId).(string)
+	return
+}

commonfile/fileproto/file_drpc.pb.go

@@ -0,0 +1,268 @@
+// Code generated by protoc-gen-go-drpc. DO NOT EDIT.
+// protoc-gen-go-drpc version: v0.0.32
+// source: commonfile/fileproto/protos/file.proto
+
+package fileproto
+
+import (
+	bytes "bytes"
+	context "context"
+	errors "errors"
+	jsonpb "github.com/gogo/protobuf/jsonpb"
+	proto "github.com/gogo/protobuf/proto"
+	drpc "storj.io/drpc"
+	drpcerr "storj.io/drpc/drpcerr"
+)
+
+type drpcEncoding_File_commonfile_fileproto_protos_file_proto struct{}
+
+func (drpcEncoding_File_commonfile_fileproto_protos_file_proto) Marshal(msg drpc.Message) ([]byte, error) {
+	return proto.Marshal(msg.(proto.Message))
+}
+
+func (drpcEncoding_File_commonfile_fileproto_protos_file_proto) Unmarshal(buf []byte, msg drpc.Message) error {
+	return proto.Unmarshal(buf, msg.(proto.Message))
+}
+
+func (drpcEncoding_File_commonfile_fileproto_protos_file_proto) JSONMarshal(msg drpc.Message) ([]byte, error) {
+	var buf bytes.Buffer
+	err := new(jsonpb.Marshaler).Marshal(&buf, msg.(proto.Message))
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func (drpcEncoding_File_commonfile_fileproto_protos_file_proto) JSONUnmarshal(buf []byte, msg drpc.Message) error {
+	return jsonpb.Unmarshal(bytes.NewReader(buf), msg.(proto.Message))
+}
+
+type DRPCFileClient interface {
+	DRPCConn() drpc.Conn
+
+	GetBlocks(ctx context.Context) (DRPCFile_GetBlocksClient, error)
+	PushBlock(ctx context.Context, in *PushBlockRequest) (*PushBlockResponse, error)
+	DeleteBlocks(ctx context.Context, in *DeleteBlocksRequest) (*DeleteBlocksResponse, error)
+	Check(ctx context.Context, in *CheckRequest) (*CheckResponse, error)
+}
+
+type drpcFileClient struct {
+	cc drpc.Conn
+}
+
+func NewDRPCFileClient(cc drpc.Conn) DRPCFileClient {
+	return &drpcFileClient{cc}
+}
+
+func (c *drpcFileClient) DRPCConn() drpc.Conn { return c.cc }
+
+func (c *drpcFileClient) GetBlocks(ctx context.Context) (DRPCFile_GetBlocksClient, error) {
+	stream, err := c.cc.NewStream(ctx, "/anyFile.File/GetBlocks", drpcEncoding_File_commonfile_fileproto_protos_file_proto{})
+	if err != nil {
+		return nil, err
+	}
+	x := &drpcFile_GetBlocksClient{stream}
+	return x, nil
+}
+
+type DRPCFile_GetBlocksClient interface {
+	drpc.Stream
+	Send(*GetBlockRequest) error
+	Recv() (*GetBlockResponse, error)
+}
+
+type drpcFile_GetBlocksClient struct {
+	drpc.Stream
+}
+
+func (x *drpcFile_GetBlocksClient) Send(m *GetBlockRequest) error {
+	return x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{})
+}
+
+func (x *drpcFile_GetBlocksClient) Recv() (*GetBlockResponse, error) {
+	m := new(GetBlockResponse)
+	if err := x.MsgRecv(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{}); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (x *drpcFile_GetBlocksClient) RecvMsg(m *GetBlockResponse) error {
+	return x.MsgRecv(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{})
+}
+
+func (c *drpcFileClient) PushBlock(ctx context.Context, in *PushBlockRequest) (*PushBlockResponse, error) {
+	out := new(PushBlockResponse)
+	err := c.cc.Invoke(ctx, "/anyFile.File/PushBlock", drpcEncoding_File_commonfile_fileproto_protos_file_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcFileClient) DeleteBlocks(ctx context.Context, in *DeleteBlocksRequest) (*DeleteBlocksResponse, error) {
+	out := new(DeleteBlocksResponse)
+	err := c.cc.Invoke(ctx, "/anyFile.File/DeleteBlocks", drpcEncoding_File_commonfile_fileproto_protos_file_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *drpcFileClient) Check(ctx context.Context, in *CheckRequest) (*CheckResponse, error) {
+	out := new(CheckResponse)
+	err := c.cc.Invoke(ctx, "/anyFile.File/Check", drpcEncoding_File_commonfile_fileproto_protos_file_proto{}, in, out)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+type DRPCFileServer interface {
+	GetBlocks(DRPCFile_GetBlocksStream) error
+	PushBlock(context.Context, *PushBlockRequest) (*PushBlockResponse, error)
+	DeleteBlocks(context.Context, *DeleteBlocksRequest) (*DeleteBlocksResponse, error)
+	Check(context.Context, *CheckRequest) (*CheckResponse, error)
+}
+
+type DRPCFileUnimplementedServer struct{}
+
+func (s *DRPCFileUnimplementedServer) GetBlocks(DRPCFile_GetBlocksStream) error {
+	return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCFileUnimplementedServer) PushBlock(context.Context, *PushBlockRequest) (*PushBlockResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCFileUnimplementedServer) DeleteBlocks(context.Context, *DeleteBlocksRequest) (*DeleteBlocksResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+func (s *DRPCFileUnimplementedServer) Check(context.Context, *CheckRequest) (*CheckResponse, error) {
+	return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
+}
+
+type DRPCFileDescription struct{}
+
+func (DRPCFileDescription) NumMethods() int { return 4 }
+
+func (DRPCFileDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
+	switch n {
+	case 0:
+		return "/anyFile.File/GetBlocks", drpcEncoding_File_commonfile_fileproto_protos_file_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return nil, srv.(DRPCFileServer).
+					GetBlocks(
+						&drpcFile_GetBlocksStream{in1.(drpc.Stream)},
+					)
+			}, DRPCFileServer.GetBlocks, true
+	case 1:
+		return "/anyFile.File/PushBlock", drpcEncoding_File_commonfile_fileproto_protos_file_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCFileServer).
+					PushBlock(
+						ctx,
+						in1.(*PushBlockRequest),
+					)
+			}, DRPCFileServer.PushBlock, true
+	case 2:
+		return "/anyFile.File/DeleteBlocks", drpcEncoding_File_commonfile_fileproto_protos_file_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCFileServer).
+					DeleteBlocks(
+						ctx,
+						in1.(*DeleteBlocksRequest),
+					)
+			}, DRPCFileServer.DeleteBlocks, true
+	case 3:
+		return "/anyFile.File/Check", drpcEncoding_File_commonfile_fileproto_protos_file_proto{},
+			func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
+				return srv.(DRPCFileServer).
+					Check(
+						ctx,
+						in1.(*CheckRequest),
+					)
+			}, DRPCFileServer.Check, true
+	default:
+		return "", nil, nil, nil, false
+	}
+}
+
+func DRPCRegisterFile(mux drpc.Mux, impl DRPCFileServer) error {
+	return mux.Register(impl, DRPCFileDescription{})
+}
+
+type DRPCFile_GetBlocksStream interface {
+	drpc.Stream
+	Send(*GetBlockResponse) error
+	Recv() (*GetBlockRequest, error)
+}
+
+type drpcFile_GetBlocksStream struct {
+	drpc.Stream
+}
+
+func (x *drpcFile_GetBlocksStream) Send(m *GetBlockResponse) error {
+	return x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{})
+}
+
+func (x *drpcFile_GetBlocksStream) Recv() (*GetBlockRequest, error) {
+	m := new(GetBlockRequest)
+	if err := x.MsgRecv(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{}); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (x *drpcFile_GetBlocksStream) RecvMsg(m *GetBlockRequest) error {
+	return x.MsgRecv(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{})
+}
+
+type DRPCFile_PushBlockStream interface {
+	drpc.Stream
+	SendAndClose(*PushBlockResponse) error
+}
+
+type drpcFile_PushBlockStream struct {
+	drpc.Stream
+}
+
+func (x *drpcFile_PushBlockStream) SendAndClose(m *PushBlockResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCFile_DeleteBlocksStream interface {
+	drpc.Stream
+	SendAndClose(*DeleteBlocksResponse) error
+}
+
+type drpcFile_DeleteBlocksStream struct {
+	drpc.Stream
+}
+
+func (x *drpcFile_DeleteBlocksStream) SendAndClose(m *DeleteBlocksResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}
+
+type DRPCFile_CheckStream interface {
+	drpc.Stream
+	SendAndClose(*CheckResponse) error
+}
+
+type drpcFile_CheckStream struct {
+	drpc.Stream
+}
+
+func (x *drpcFile_CheckStream) SendAndClose(m *CheckResponse) error {
+	if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_protos_file_proto{}); err != nil {
+		return err
+	}
+	return x.CloseSend()
+}

commonfile/fileproto/fileprotoerr/fileprotoerr.go

@@ -0,0 +1,13 @@
+package fileprotoerr
+
+import (
+	"fmt"
+	"github.com/anytypeio/any-sync/commonfile/fileproto"
+	"github.com/anytypeio/any-sync/net/rpc/rpcerr"
+)
+
+var (
+	errGroup       = rpcerr.ErrGroup(fileproto.ErrCodes_ErrorOffset)
+	ErrUnexpected  = errGroup.Register(fmt.Errorf("unexpected fileproto error"), uint64(fileproto.ErrCodes_Unexpected))
+	ErrCIDNotFound = errGroup.Register(fmt.Errorf("CID not found"), uint64(fileproto.ErrCodes_CIDNotFound))
+)

commonfile/fileproto/protos/file.proto

@@ -0,0 +1,60 @@
+syntax = "proto3";
+package anyFile;
+
+option go_package = "commonfile/fileproto";
+
+enum ErrCodes {
+    Unexpected = 0;
+    CIDNotFound = 1;
+    ErrorOffset = 200;
+}
+
+service File {
+    // GetBlocks streams ipfs blocks from server to client
+    rpc GetBlocks(stream GetBlockRequest) returns (stream GetBlockResponse);
+    // PushBlock pushes one block to server
+    rpc PushBlock(PushBlockRequest) returns (PushBlockResponse);
+    // DeleteBlock deletes block from space
+    rpc DeleteBlocks(DeleteBlocksRequest) returns (DeleteBlocksResponse);
+    // Ping checks the connection
+    rpc Check(CheckRequest) returns (CheckResponse);
+}
+
+message GetBlockRequest {
+    string spaceId = 1;
+    bytes cid = 2;
+}
+
+message GetBlockResponse {
+    bytes cid = 1;
+    bytes data = 2;
+    CIDError code = 3;
+}
+
+message PushBlockRequest {
+    string spaceId = 1;
+    bytes cid = 2;
+    bytes data = 3;
+}
+
+message PushBlockResponse {}
+
+message DeleteBlocksRequest {
+    string spaceId = 1;
+    repeated bytes cid = 2;
+}
+
+message DeleteBlocksResponse {}
+
+message CheckRequest {}
+
+message CheckResponse {
+    repeated string spaceIds = 1;
+}
+
+
+enum CIDError {
+    CIDErrorOk = 0;
+    CIDErrorNotFound = 1;
+    CIDErrorUnexpected = 2;
+}

commonfile/fileservice/blockservice.go

@@ -0,0 +1,51 @@
+package fileservice
+
+import (
+	"context"
+	"github.com/anytypeio/any-sync/commonfile/fileblockstore"
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	exchange "github.com/ipfs/go-ipfs-exchange-interface"
+)
+
+func newBlockService(store fileblockstore.BlockStore) blockservice.BlockService {
+	return &blockService{store: store}
+}
+
+type blockService struct {
+	store fileblockstore.BlockStore
+}
+
+func (bs *blockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	return bs.store.Get(ctx, c)
+}
+
+func (bs *blockService) GetBlocks(ctx context.Context, ks []cid.Cid) <-chan blocks.Block {
+	return bs.store.GetMany(ctx, ks)
+}
+
+func (bs *blockService) Blockstore() blockstore.Blockstore {
+	return nil
+}
+
+func (bs *blockService) Exchange() exchange.Interface {
+	return nil
+}
+
+func (bs *blockService) AddBlock(ctx context.Context, b blocks.Block) error {
+	return bs.store.Add(ctx, []blocks.Block{b})
+}
+
+func (bs *blockService) AddBlocks(ctx context.Context, b []blocks.Block) error {
+	return bs.store.Add(ctx, b)
+}
+
+func (bs *blockService) DeleteBlock(ctx context.Context, c cid.Cid) error {
+	return bs.store.Delete(ctx, c)
+}
+
+func (bs *blockService) Close() error {
+	return nil
+}

commonfile/fileservice/fileservice.go

@@ -0,0 +1,88 @@
+package fileservice
+
+import (
+	"context"
+	"fmt"
+	"github.com/anytypeio/any-sync/app"
+	"github.com/anytypeio/any-sync/app/logger"
+	"github.com/anytypeio/any-sync/commonfile/fileblockstore"
+	"github.com/ipfs/go-cid"
+	chunker "github.com/ipfs/go-ipfs-chunker"
+	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
+	"github.com/ipfs/go-unixfs/importer/balanced"
+	"github.com/ipfs/go-unixfs/importer/helpers"
+	ufsio "github.com/ipfs/go-unixfs/io"
+	"github.com/multiformats/go-multihash"
+	"go.uber.org/zap"
+	"io"
+)
+
+const CName = "common.commonfile.fileservice"
+
+var log = logger.NewNamed(CName)
+
+func New() FileService {
+	return &fileService{}
+}
+
+type FileService interface {
+	// GetFile gets file from ipfs storage
+	GetFile(ctx context.Context, c cid.Cid) (ufsio.ReadSeekCloser, error)
+	// AddFile adds file to ipfs storage
+	AddFile(ctx context.Context, r io.Reader) (ipld.Node, error)
+	app.Component
+}
+
+type fileService struct {
+	merkledag ipld.DAGService
+	prefix    cid.Prefix
+}
+
+func (fs *fileService) Init(a *app.App) (err error) {
+	prefix, err := merkledag.PrefixForCidVersion(1)
+	if err != nil {
+		return fmt.Errorf("bad CID Version: %s", err)
+	}
+	hashFunCode, ok := multihash.Names["sha2-256"]
+	if !ok {
+		return fmt.Errorf("unrecognized hash function")
+	}
+	prefix.MhType = hashFunCode
+	prefix.MhLength = -1
+	bs := newBlockService(a.MustComponent(fileblockstore.CName).(fileblockstore.BlockStore))
+	fs.merkledag = merkledag.NewDAGService(bs)
+	fs.prefix = prefix
+	return
+}
+
+func (fs *fileService) Name() string {
+	return CName
+}
+
+func (fs *fileService) AddFile(ctx context.Context, r io.Reader) (ipld.Node, error) {
+	dbp := helpers.DagBuilderParams{
+		Dagserv:    fs.merkledag,
+		Maxlinks:   helpers.DefaultLinksPerBlock,
+		CidBuilder: &fs.prefix,
+	}
+	dbh, err := dbp.New(chunker.DefaultSplitter(r))
+	if err != nil {
+		return nil, err
+	}
+	n, err := balanced.Layout(dbh)
+	if err != nil {
+		return nil, err
+	}
+	log.Debug("add file", zap.String("cid", n.Cid().String()))
+	return n, nil
+}
+
+func (fs *fileService) GetFile(ctx context.Context, c cid.Cid) (ufsio.ReadSeekCloser, error) {
+	log.Debug("get file", zap.String("cid", c.String()))
+	n, err := fs.merkledag.Get(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+	return ufsio.NewDagReader(ctx, n, fs.merkledag)
+}