From 054e3635f39f7285ca6c10c7746321c57bbbaa36 Mon Sep 17 00:00:00 2001 From: Roman Godmaire Date: Sat, 10 Feb 2024 14:44:10 -0500 Subject: [PATCH] feat: switch to local filestore The plan is to adopt a sidecar model where files are written to disk then transcoded by a sidecar written in rust where they will then be written to another location (ie. Backblaze B2, S3, NFS) --- plice/src/app.d.ts | 5 ++-- plice/src/hooks.server.ts | 16 ++++++------ plice/src/lib/server/storage/index.ts | 8 ++++-- plice/src/lib/server/storage/local.ts | 29 ++++++++++++++++++++++ plice/src/lib/server/storage/noop.ts | 12 ++++++--- plice/src/lib/server/storage/s3.ts | 33 ------------------------- plice/src/routes/upload/+page.server.ts | 8 +++--- 7 files changed, 59 insertions(+), 52 deletions(-) create mode 100644 plice/src/lib/server/storage/local.ts delete mode 100644 plice/src/lib/server/storage/s3.ts diff --git a/plice/src/app.d.ts b/plice/src/app.d.ts index bba5ac7..a4d6778 100644 --- a/plice/src/app.d.ts +++ b/plice/src/app.d.ts @@ -1,6 +1,6 @@ import type { Auth, AuthRequest } from 'lucia'; import type { Database } from '$lib/server/db'; -import type { ObjectStorage } from '$lib/server/storage'; +import type { ReadFileStore, WriteFileStore } from '$lib/server/storage'; declare global { namespace App { @@ -8,7 +8,8 @@ declare global { interface Locals { database: Database; auth: Auth; - objectStorage: ObjectStorage; + readStore: ReadFileStore; + writeStore: WriteFileStore; authReq: AuthRequest; } diff --git a/plice/src/hooks.server.ts b/plice/src/hooks.server.ts index 8b89ba1..014cefc 100644 --- a/plice/src/hooks.server.ts +++ b/plice/src/hooks.server.ts @@ -3,18 +3,20 @@ import type { Handle } from '@sveltejs/kit'; import { env } from '$env/dynamic/private'; import { auth } from '$lib/server/lucia'; -import { ObjectStorageNoop } from '$lib/server/storage/noop'; +import { LocalFileStore } from '$lib/server/storage/local'; import { DatabasePrisma } from '$lib/server/db/prisma'; -const s3Client = new ObjectStorageNoop(); +const localFileStore = new LocalFileStore(env.FILE_BASE_PATH, env.FILE_EXTENSION); const prismaClient = new DatabasePrisma(env.DATABASE_URL); export const handle: Handle = async ({ event, resolve }) => { - event.locals.auth = auth; - event.locals.database = prismaClient; - event.locals.objectStorage = s3Client; - - event.locals.authReq = auth.handleRequest(event); + event.locals = { + auth, + authReq: auth.handleRequest(event), + database: prismaClient, + writeStore: localFileStore, + readStore: localFileStore + }; return await resolve(event); }; diff --git a/plice/src/lib/server/storage/index.ts b/plice/src/lib/server/storage/index.ts index f369f2c..11ebb11 100644 --- a/plice/src/lib/server/storage/index.ts +++ b/plice/src/lib/server/storage/index.ts @@ -1,3 +1,7 @@ -export interface ObjectStorage { - putObject: (key: string, obj: Uint8Array) => Promise; +export interface ReadFileStore { + read: (key: string) => Promise; +} + +export interface WriteFileStore { + write: (key: string, obj: Buffer) => Promise; } diff --git a/plice/src/lib/server/storage/local.ts b/plice/src/lib/server/storage/local.ts new file mode 100644 index 0000000..a96ce40 --- /dev/null +++ b/plice/src/lib/server/storage/local.ts @@ -0,0 +1,29 @@ +import type { ReadFileStore, WriteFileStore } from '.'; +import { promises as fs } from 'node:fs'; + +export class LocalFileStore implements ReadFileStore, WriteFileStore { + basePath: string; + extension: string; + + constructor(basePath: string, extension: string) { + this.basePath = basePath; + this.extension = extension; + } + + private getPath = (key: string) => { + return `${this.basePath}/${key}.${this.extension}`; + }; + + write = async (key: string, obj: Buffer) => { + const path = this.getPath(key); + await fs.writeFile(path, obj); + + return null; + }; + + read = async (key: string) => { + const path = this.getPath(key); + const obj = await fs.readFile(path); + return obj; + }; +} diff --git a/plice/src/lib/server/storage/noop.ts b/plice/src/lib/server/storage/noop.ts index 890b708..19c3b03 100644 --- a/plice/src/lib/server/storage/noop.ts +++ b/plice/src/lib/server/storage/noop.ts @@ -1,9 +1,13 @@ -import type { ObjectStorage } from '.'; +import type { ReadFileStore, WriteFileStore } from '.'; -class ObjectStorageNoop implements ObjectStorage { - putObject = async (_key: string, _obj: Uint8Array) => { +class NoopFileStore implements ReadFileStore, WriteFileStore { + read = async (_key: string) => { + return new ArrayBuffer(0) as Buffer; + }; + + write = async (_key: string, _obj: Buffer) => { return null; }; } -export { ObjectStorageNoop }; +export { NoopFileStore }; diff --git a/plice/src/lib/server/storage/s3.ts b/plice/src/lib/server/storage/s3.ts deleted file mode 100644 index e358035..0000000 --- a/plice/src/lib/server/storage/s3.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { ObjectStorage } from '.'; -import { PutObjectCommand, S3Client, S3ServiceException } from '@aws-sdk/client-s3'; - -class ObjectStorageS3 implements ObjectStorage { - client: S3Client; - bucket: string; - - constructor(url: string, bucket: string) { - this.bucket = bucket; - this.client = new S3Client({ - endpoint: url, - region: 'us-east-1' - }); - } - - putObject = async (key: string, obj: Uint8Array) => { - const command = new PutObjectCommand({ - Bucket: this.bucket, - Key: key, - Body: obj - }); - - try { - await this.client.send(command); - } catch (err) { - return err as S3ServiceException; - } - - return null; - }; -} - -export { ObjectStorageS3 }; diff --git a/plice/src/routes/upload/+page.server.ts b/plice/src/routes/upload/+page.server.ts index e3c428a..2cb0943 100644 --- a/plice/src/routes/upload/+page.server.ts +++ b/plice/src/routes/upload/+page.server.ts @@ -10,7 +10,7 @@ export const load: PageServerLoad = async ({ locals: { authReq } }) => { }; export const actions: Actions = { - default: async ({ request, locals: { authReq, database, objectStorage } }) => { + default: async ({ request, locals: { authReq, database, writeStore } }) => { const session = await authReq.validate(); if (!session) redirect(302, '/login'); @@ -20,9 +20,9 @@ export const actions: Actions = { const title = formData.get('title') as string; const file = formData.get('file') as File; - const objectKey = `${uuidv4()}.mp3`; + const objectKey = `${uuidv4()}`; - const err = await objectStorage.putObject(objectKey, new Uint8Array(await file.arrayBuffer())); + const err = await writeStore.write(objectKey, (await file.arrayBuffer()) as Buffer); if (err) { console.log(err); return fail(500, { @@ -30,7 +30,7 @@ export const actions: Actions = { }); } - const track = await database.createTrack(producerId, title, objectKey); + const track = await database.createTrack(producerId, title); await database.createTrackVersion(track.id); redirect(302, '/');