From 3832c2b1b9a8c78dd4149b5c78fd4558ff56366e Mon Sep 17 00:00:00 2001 From: Tanner Sommers Date: Mon, 16 Sep 2024 15:39:36 -0500 Subject: [PATCH] :3 --- api/package.json | 2 +- api/src/core/api.js | 283 ++++++++++++++-------------- api/src/stream/manage.js | 84 ++++----- api/src/{cobalt.js => thvideodl.js} | 0 4 files changed, 174 insertions(+), 195 deletions(-) rename api/src/{cobalt.js => thvideodl.js} (100%) diff --git a/api/package.json b/api/package.json index 2330b8a..887a3d1 100644 --- a/api/package.json +++ b/api/package.json @@ -9,7 +9,7 @@ "node": ">=18" }, "scripts": { - "start": "node src/cobalt", + "start": "node src/thvideodl", "setup": "node src/util/setup", "test": "node src/util/test", "token:youtube": "node src/util/generate-youtube-tokens" diff --git a/api/src/core/api.js b/api/src/core/api.js index b1ad25b..1ed8ed4 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -1,43 +1,54 @@ -import cors from "cors"; -import rateLimit from "express-rate-limit"; -import { setGlobalDispatcher, ProxyAgent } from "undici"; -import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info"; - -import jwt from "../security/jwt.js"; -import stream from "../stream/stream.js"; -import match from "../processing/match.js"; - -import { env } from "../config.js"; -import { extract } from "../processing/url.js"; -import { languageCode } from "../misc/utils.js"; -import { Bright, Cyan } from "../misc/console-text.js"; -import { generateHmac, generateSalt } from "../misc/crypto.js"; -import { randomizeCiphers } from "../misc/randomize-ciphers.js"; -import { verifyTurnstileToken } from "../security/turnstile.js"; -import { friendlyServiceName } from "../processing/service-alias.js"; -import { verifyStream, getInternalStream } from "../stream/manage.js"; -import { createResponse, normalizeRequest, getIP } from "../processing/request.js"; +import cors from 'cors'; +import rateLimit from 'express-rate-limit'; +import { setGlobalDispatcher, ProxyAgent } from 'undici'; +import { + getCommit, + getBranch, + getRemote, + getVersion, +} from '@imput/version-info'; + +import jwt from '../security/jwt.js'; +import stream from '../stream/stream.js'; +import match from '../processing/match.js'; + +import { env } from '../config.js'; +import { extract } from '../processing/url.js'; +import { languageCode } from '../misc/utils.js'; +import { Bright, Cyan } from '../misc/console-text.js'; +import { generateHmac, generateSalt } from '../misc/crypto.js'; +import { randomizeCiphers } from '../misc/randomize-ciphers.js'; +import { verifyTurnstileToken } from '../security/turnstile.js'; +import { friendlyServiceName } from '../processing/service-alias.js'; +import { verifyStream, getInternalStream } from '../stream/manage.js'; +import { + createResponse, + normalizeRequest, + getIP, +} from '../processing/request.js'; const git = { branch: await getBranch(), commit: await getCommit(), remote: await getRemote(), -} +}; const version = await getVersion(); const acceptRegex = /^application\/json(; charset=utf-8)?$/; const ipSalt = generateSalt(); -const corsConfig = env.corsWildcard ? {} : { - origin: env.corsURL, - optionsSuccessStatus: 200 -} +const corsConfig = env.corsWildcard + ? {} + : { + origin: env.corsURL, + optionsSuccessStatus: 200, + }; const fail = (res, code, context) => { - const { status, body } = createResponse("error", { code, context }); + const { status, body } = createResponse('error', { code, context }); res.status(status).json(body); -} +}; export const runAPI = (express, app, __dirname) => { const startTime = new Date(); @@ -49,96 +60,100 @@ export const runAPI = (express, app, __dirname) => { url: env.apiURL, startTime: `${startTimestamp}`, durationLimit: env.durationLimit, - services: [...env.enabledServices].map(e => { + services: [...env.enabledServices].map((e) => { return friendlyServiceName(e); }), }, git, - }) + }); const apiLimiter = rateLimit({ windowMs: env.rateLimitWindow * 1000, max: env.rateLimitMax, standardHeaders: true, legacyHeaders: false, - keyGenerator: req => { + keyGenerator: (req) => { if (req.authorized) { - return generateHmac(req.header("Authorization"), ipSalt); + return generateHmac(req.header('Authorization'), ipSalt); } return generateHmac(getIP(req), ipSalt); }, handler: (req, res) => { - const { status, body } = createResponse("error", { - code: "error.api.rate_exceeded", + const { status, body } = createResponse('error', { + code: 'error.api.rate_exceeded', context: { - limit: env.rateLimitWindow - } + limit: env.rateLimitWindow, + }, }); return res.status(status).json(body); - } - }) + }, + }); const apiLimiterStream = rateLimit({ windowMs: env.rateLimitWindow * 1000, max: env.rateLimitMax, standardHeaders: true, legacyHeaders: false, - keyGenerator: req => generateHmac(getIP(req), ipSalt), + keyGenerator: (req) => generateHmac(getIP(req), ipSalt), handler: (req, res) => { - return res.sendStatus(429) - } - }) + return res.sendStatus(429); + }, + }); app.set('trust proxy', ['loopback', 'uniquelocal']); - app.use('/', cors({ - methods: ['GET', 'POST'], - exposedHeaders: [ - 'Ratelimit-Limit', - 'Ratelimit-Policy', - 'Ratelimit-Remaining', - 'Ratelimit-Reset' - ], - ...corsConfig, - })); + app.use( + '/', + cors({ + methods: ['GET', 'POST'], + exposedHeaders: [ + 'Ratelimit-Limit', + 'Ratelimit-Policy', + 'Ratelimit-Remaining', + 'Ratelimit-Reset', + ], + ...corsConfig, + }), + ); app.post('/', apiLimiter); app.use('/tunnel', apiLimiterStream); app.post('/', (req, res, next) => { if (!env.turnstileSecret || !env.jwtSecret) { + req.authorized = true; // bypass auth if not configured return next(); } try { - const authorization = req.header("Authorization"); - if (!authorization) { - return fail(res, "error.api.auth.jwt.missing"); - } + // const authorization = req.header("Authorization"); + // if (!authorization) { + // return fail(res, "error.api.auth.jwt.missing"); + // } - if (!authorization.startsWith("Bearer ") || authorization.length > 256) { - return fail(res, "error.api.auth.jwt.invalid"); - } + // if (!authorization.startsWith("Bearer ") || authorization.length > 256) { + // return fail(res, "error.api.auth.jwt.invalid"); + // } - const verifyJwt = jwt.verify( - authorization.split("Bearer ", 2)[1] - ); + // const verifyJwt = jwt.verify( + // authorization.split("Bearer ", 2)[1] + // ); - if (!verifyJwt) { - return fail(res, "error.api.auth.jwt.invalid"); - } + // if (!verifyJwt) { + // return fail(res, "error.api.auth.jwt.invalid"); + // } if (!acceptRegex.test(req.header('Accept'))) { - return fail(res, "error.api.header.accept"); + return fail(res, 'error.api.header.accept'); } if (!acceptRegex.test(req.header('Content-Type'))) { - return fail(res, "error.api.header.content_type"); + return fail(res, 'error.api.header.content_type'); } req.authorized = true; } catch { - return fail(res, "error.api.generic"); + return fail(res, 'error.api.generic'); } next(); }); @@ -146,8 +161,8 @@ export const runAPI = (express, app, __dirname) => { app.use('/', express.json({ limit: 1024 })); app.use('/', (err, _, res, next) => { if (err) { - const { status, body } = createResponse("error", { - code: "error.api.invalid_body", + const { status, body } = createResponse('error', { + code: 'error.api.invalid_body', }); return res.status(status).json(body); } @@ -155,56 +170,30 @@ export const runAPI = (express, app, __dirname) => { next(); }); - app.post("/session", async (req, res) => { - if (!env.turnstileSecret || !env.jwtSecret) { - return fail(res, "error.api.auth.not_configured") - } - - const turnstileResponse = req.header("cf-turnstile-response"); - - if (!turnstileResponse) { - return fail(res, "error.api.auth.turnstile.missing"); - } - - const turnstileResult = await verifyTurnstileToken( - turnstileResponse, - req.ip - ); - - if (!turnstileResult) { - return fail(res, "error.api.auth.turnstile.invalid"); - } - - try { - res.json(jwt.generate()); - } catch { - return fail(res, "error.api.generic"); - } - }); - app.post('/', async (req, res) => { const request = req.body; const lang = languageCode(req); if (!request.url) { - return fail(res, "error.api.link.missing"); + return fail(res, 'error.api.link.missing'); } if (request.youtubeDubBrowserLang) { request.youtubeDubLang = lang; } - const { success, data: normalizedRequest } = await normalizeRequest(request); + const { success, data: normalizedRequest } = + await normalizeRequest(request); if (!success) { - return fail(res, "error.api.invalid_body"); + return fail(res, 'error.api.invalid_body'); } const parsed = extract(normalizedRequest.url); if (!parsed) { - return fail(res, "error.api.link.invalid"); + return fail(res, 'error.api.link.invalid'); } - if ("error" in parsed) { + if ('error' in parsed) { let context; if (parsed?.context) { context = parsed.context; @@ -221,30 +210,19 @@ export const runAPI = (express, app, __dirname) => { res.status(result.status).json(result.body); } catch { - fail(res, "error.api.generic"); + fail(res, 'error.api.generic'); } - }) + }); app.get('/tunnel', (req, res) => { const id = String(req.query.id); - const exp = String(req.query.exp); - const sig = String(req.query.sig); - const sec = String(req.query.sec); - const iv = String(req.query.iv); - - const checkQueries = id && exp && sig && sec && iv; - const checkBaseLength = id.length === 21 && exp.length === 13; - const checkSafeLength = sig.length === 43 && sec.length === 43 && iv.length === 22; + const exp = Number(req.query.exp); - if (!checkQueries || !checkBaseLength || !checkSafeLength) { + if (!id || !exp) { return res.status(400).end(); } - if (req.query.p) { - return res.status(200).end(); - } - - const streamInfo = verifyStream(id, sig, exp, sec, iv); + const streamInfo = verifyStream(id, exp); if (!streamInfo?.service) { return res.status(streamInfo.status).end(); } @@ -254,7 +232,7 @@ export const runAPI = (express, app, __dirname) => { } return stream(res, streamInfo); - }) + }); app.get('/itunnel', (req, res) => { if (!req.ip.endsWith('127.0.0.1')) { @@ -272,55 +250,72 @@ export const runAPI = (express, app, __dirname) => { streamInfo.headers = new Map([ ...(streamInfo.headers || []), - ...Object.entries(req.headers) + ...Object.entries(req.headers), ]); return stream(res, { type: 'internal', ...streamInfo }); - }) + }); app.get('/', (_, res) => { res.type('json'); res.status(200).send(serverInfo); - }) + }); app.get('/favicon.ico', (req, res) => { res.status(404).end(); - }) + }); app.get('/*', (req, res) => { res.redirect('/'); - }) + }); // handle all express errors app.use((_, __, res, ___) => { - return fail(res, "error.api.generic"); - }) + return fail(res, 'error.api.generic'); + }); - randomizeCiphers(); - setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes + // randomizeCiphers(); + // setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes if (env.externalProxy) { if (env.freebindCIDR) { - throw new Error('Freebind is not available when external proxy is enabled') + throw new Error( + 'Freebind is not available when external proxy is enabled', + ); } - setGlobalDispatcher(new ProxyAgent(env.externalProxy)) + setGlobalDispatcher(new ProxyAgent(env.externalProxy)); } app.listen(env.apiPort, env.listenAddress, () => { - console.log(`\n` + - Bright(Cyan("Team Hydra Video DL")) + Bright("API :3") + "\n" + - - "~~~~~~\n" + - Bright("version: ") + version + "\n" + - Bright("commit: ") + git.commit + "\n" + - Bright("branch: ") + git.branch + "\n" + - Bright("remote: ") + git.remote + "\n" + - Bright("start time: ") + startTime.toUTCString() + "\n" + - "~~~~~~\n" + - - Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" + - Bright("port: ") + env.apiPort + "\n" - ) - }) -} + console.log( + `\n` + + Bright(Cyan('Team Hydra Video DL')) + + Bright('API :3') + + '\n' + + '~~~~~~\n' + + Bright('version: ') + + version + + '\n' + + Bright('commit: ') + + git.commit + + '\n' + + Bright('branch: ') + + git.branch + + '\n' + + Bright('remote: ') + + git.remote + + '\n' + + Bright('start time: ') + + startTime.toUTCString() + + '\n' + + '~~~~~~\n' + + Bright('url: ') + + Bright(Cyan(env.apiURL)) + + '\n' + + Bright('port: ') + + env.apiPort + + '\n', + ); + }); +}; diff --git a/api/src/stream/manage.js b/api/src/stream/manage.js index e25f443..81f7a06 100644 --- a/api/src/stream/manage.js +++ b/api/src/stream/manage.js @@ -1,36 +1,33 @@ -import NodeCache from "node-cache"; +import NodeCache from 'node-cache'; -import { nanoid } from "nanoid"; -import { randomBytes } from "crypto"; -import { strict as assert } from "assert"; -import { setMaxListeners } from "node:events"; +import { nanoid } from 'nanoid'; +import { randomBytes } from 'crypto'; +import { strict as assert } from 'assert'; +import { setMaxListeners } from 'node:events'; -import { env } from "../config.js"; -import { closeRequest } from "./shared.js"; -import { decryptStream, encryptStream, generateHmac } from "../misc/crypto.js"; +import { env } from '../config.js'; +import { closeRequest } from './shared.js'; +import { decryptStream, encryptStream, generateHmac } from '../misc/crypto.js'; // optional dependency -const freebind = env.freebindCIDR && await import('freebind').catch(() => {}); +const freebind = env.freebindCIDR && (await import('freebind').catch(() => {})); const streamCache = new NodeCache({ stdTTL: env.streamLifespan, checkperiod: 10, - deleteOnExpire: true -}) + deleteOnExpire: true, +}); -streamCache.on("expired", (key) => { +streamCache.on('expired', (key) => { streamCache.del(key); -}) +}); const internalStreamCache = new Map(); const hmacSalt = randomBytes(64).toString('hex'); export function createStream(obj) { const streamID = nanoid(), - iv = randomBytes(16).toString('base64url'), - secret = randomBytes(32).toString('base64url'), exp = new Date().getTime() + env.streamLifespan * 1000, - hmac = generateHmac(`${streamID},${exp},${iv},${secret}`, hmacSalt), streamData = { exp: exp, type: obj.type, @@ -48,20 +45,13 @@ export function createStream(obj) { audioFormat: obj.audioFormat, }; - streamCache.set( - streamID, - encryptStream(streamData, iv, secret) - ) - + streamCache.set(streamID, streamData); let streamLink = new URL('/tunnel', env.apiURL); const params = { - 'id': streamID, - 'exp': exp, - 'sig': hmac, - 'sec': secret, - 'iv': iv - } + id: streamID, + exp: exp, + }; for (const [key, value] of Object.entries(params)) { streamLink.searchParams.append(key, value); @@ -79,7 +69,9 @@ export function createInternalStream(url, obj = {}) { let dispatcher; if (obj.requestIP) { - dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { strict: false }) + dispatcher = freebind?.dispatcherFromIP(obj.requestIP, { + strict: false, + }); } const streamID = nanoid(); @@ -100,7 +92,7 @@ export function createInternalStream(url, obj = {}) { service: obj.service, headers, controller, - dispatcher + dispatcher, }); let streamLink = new URL('/itunnel', `http://127.0.0.1:${env.apiPort}`); @@ -109,7 +101,7 @@ export function createInternalStream(url, obj = {}) { const cleanup = () => { destroyInternalStream(streamLink); controller.signal.removeEventListener('abort', cleanup); - } + }; controller.signal.addEventListener('abort', cleanup); @@ -134,11 +126,14 @@ function wrapStream(streamInfo) { const url = streamInfo.urls; if (typeof url === 'string') { + console.log(`[wrapStream] url is a string`); streamInfo.urls = createInternalStream(url, streamInfo); } else if (Array.isArray(url)) { + console.log(`[wrapStream] url is an array`); for (const idx in streamInfo.urls) { streamInfo.urls[idx] = createInternalStream( - streamInfo.urls[idx], streamInfo + streamInfo.urls[idx], + streamInfo, ); } } else throw 'invalid urls'; @@ -146,24 +141,13 @@ function wrapStream(streamInfo) { return streamInfo; } -export function verifyStream(id, hmac, exp, secret, iv) { - try { - const ghmac = generateHmac(`${id},${exp},${iv},${secret}`, hmacSalt); - const cache = streamCache.get(id.toString()); - - if (ghmac !== String(hmac)) return { status: 401 }; - if (!cache) return { status: 404 }; - - const streamInfo = JSON.parse(decryptStream(cache, iv, secret)); +export function verifyStream(id, exp) { + console.log(`[verifyStream] id: ${id}`); + const streamInfo = streamCache.get(id.toString()); + console.log(`[verifyStream] streamInfo: ${streamInfo}`); + if (!streamInfo) return { status: 404 }; - if (!streamInfo) return { status: 404 }; - - if (Number(exp) <= new Date().getTime()) - return { status: 404 }; - - return wrapStream(streamInfo); - } - catch { - return { status: 500 }; - } + if (Number(exp) <= new Date().getTime()) return { status: 404 }; + console.log(`[verifyStream] stream is OK! Returning streamInfo`); + return wrapStream(streamInfo); } diff --git a/api/src/cobalt.js b/api/src/thvideodl.js similarity index 100% rename from api/src/cobalt.js rename to api/src/thvideodl.js