From 9d629ce34ebc2f7e86f207fdf589c65808afac3e Mon Sep 17 00:00:00 2001 From: Sticks Date: Fri, 21 Mar 2025 22:12:09 -0400 Subject: [PATCH] sync upstream (api@0a7cf75) --- api/package.json | 5 +- api/src/config.js | 12 ++ api/src/core/api.js | 8 +- api/src/processing/cookie/manager.js | 1 - api/src/processing/match.js | 2 +- api/src/processing/services/pinterest.js | 2 +- api/src/processing/services/twitter.js | 130 +++++++++++++++++----- api/src/processing/services/youtube.js | 133 ++++++++++------------- api/src/util/tests/twitter.json | 9 ++ api/src/util/tests/xiaohongshu.json | 15 ++- api/src/util/tests/youtube.json | 4 + 11 files changed, 203 insertions(+), 118 deletions(-) diff --git a/api/package.json b/api/package.json index fc5304e..48f75d1 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.7.7", + "version": "10.8.2", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -11,7 +11,6 @@ "scripts": { "start": "node src/cobalt", "test": "node src/util/test", - "token:youtube": "node src/util/generate-youtube-tokens", "token:jwt": "node src/util/generate-jwt-secret" }, "repository": { @@ -39,7 +38,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^13.1.0", + "youtubei.js": "^13.2.0", "zod": "^3.23.8" }, "optionalDependencies": { diff --git a/api/src/config.js b/api/src/config.js index 191e844..98da6fe 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,3 +1,4 @@ +import { Constants } from "youtubei.js"; import { getVersion } from "@imput/version-info"; import { services } from "./processing/service-config.js"; import { supportsReusePort } from "./misc/cluster.js"; @@ -52,6 +53,11 @@ const env = { keyReloadInterval: 900, enabledServices, + + customInnertubeClient: process.env.CUSTOM_INNERTUBE_CLIENT, + ytSessionServer: process.env.YOUTUBE_SESSION_SERVER, + ytSessionReloadInterval: 300, + ytSessionInnertubeClient: process.env.YOUTUBE_SESSION_INNERTUBE_CLIENT, } const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; @@ -74,6 +80,12 @@ if (env.instanceCount > 1 && !env.redisURL) { throw new Error('SO_REUSEPORT is not supported'); } +if (env.customInnertubeClient && !Constants.SUPPORTED_CLIENTS.includes(env.customInnertubeClient)) { + console.error("CUSTOM_INNERTUBE_CLIENT is invalid. Provided client is not supported."); + console.error(`Supported clients are: ${Constants.SUPPORTED_CLIENTS.join(', ')}\n`); + throw new Error("Invalid CUSTOM_INNERTUBE_CLIENT"); +} + export { env, genericUserAgent, diff --git a/api/src/core/api.js b/api/src/core/api.js index e4d3dfc..82449c3 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -18,8 +18,10 @@ import { verifyTurnstileToken } from "../security/turnstile.js"; import { friendlyServiceName } from "../processing/service-alias.js"; import { verifyStream, getInternalStream } from "../stream/manage.js"; import { createResponse, normalizeRequest, getIP } from "../processing/request.js"; + import * as APIKeys from "../security/api-keys.js"; import * as Cookies from "../processing/cookie/manager.js"; +import * as YouTubeSession from "../processing/helpers/youtube-session.js"; const git = { branch: await getBranch(), @@ -354,7 +356,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { }, () => { if (isPrimary) { console.log(`\n` + - Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" + + Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" + "~~~~~~\n" + Bright("version: ") + version + "\n" + @@ -376,6 +378,10 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { if (env.cookiePath) { Cookies.setup(env.cookiePath); } + + if (env.ytSessionServer) { + YouTubeSession.setup(); + } }); if (isCluster) { diff --git a/api/src/processing/cookie/manager.js b/api/src/processing/cookie/manager.js index 25f41c2..9e23374 100644 --- a/api/src/processing/cookie/manager.js +++ b/api/src/processing/cookie/manager.js @@ -13,7 +13,6 @@ const VALID_SERVICES = new Set([ 'reddit', 'twitter', 'youtube', - 'youtube_oauth' ]); const invalidCookies = {}; diff --git a/api/src/processing/match.js b/api/src/processing/match.js index e2d6aa0..ee4fdc1 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -109,7 +109,7 @@ export default async function({ host, patternMatch, params }) { } if (url.hostname === "music.youtube.com" || isAudioOnly) { - fetchInfo.quality = "max"; + fetchInfo.quality = "1080"; fetchInfo.format = "vp9"; fetchInfo.isAudioOnly = true; fetchInfo.isAudioMuted = false; diff --git a/api/src/processing/services/pinterest.js b/api/src/processing/services/pinterest.js index ea4275c..15566cc 100644 --- a/api/src/processing/services/pinterest.js +++ b/api/src/processing/services/pinterest.js @@ -23,7 +23,7 @@ export default async function(o) { const videoLink = [...html.matchAll(videoRegex)] .map(([, link]) => link) - .find(a => a.endsWith('.mp4') && a.includes('720p')); + .find(a => a.endsWith('.mp4')); if (videoLink) return { urls: videoLink, diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index b4a1d55..a4f4505 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -24,6 +24,11 @@ const badContainerEnd = new Date(1702605600000); function needsFixing(media) { const representativeId = media.source_status_id_str ?? media.id_str; + + // syndication api doesn't have media ids in its response, + // so we just assume it's all good + if (!representativeId) return false; + const mediaTimestamp = new Date( Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH) ); @@ -53,6 +58,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => { } } +const requestSyndication = async(dispatcher, tweetId) => { + // thank you + // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334 + const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, ''); + const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result"); + + syndicationUrl.searchParams.set("id", tweetId); + syndicationUrl.searchParams.set("token", token(tweetId)); + + const result = await fetch(syndicationUrl, { + headers: { + "user-agent": genericUserAgent + }, + dispatcher + }); + + return result; +} + const requestTweet = async(dispatcher, tweetId, token, cookie) => { const graphqlTweetURL = new URL(graphqlURL); @@ -87,36 +111,24 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => { let result = await fetch(graphqlTweetURL, { headers, dispatcher }); updateCookie(cookie, result.headers); - // we might have been missing the `ct0` cookie, retry + // we might have been missing the ct0 cookie, retry if (result.status === 403 && result.headers.get('set-cookie')) { - result = await fetch(graphqlTweetURL, { - headers: { - ...headers, - 'x-csrf-token': cookie.values().ct0 - }, - dispatcher - }); + const cookieValues = cookie?.values(); + if (cookieValues?.ct0) { + result = await fetch(graphqlTweetURL, { + headers: { + ...headers, + 'x-csrf-token': cookieValues.ct0 + }, + dispatcher + }); + } } return result } -export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { - const cookie = await getCookie('twitter'); - - let guestToken = await getGuestToken(dispatcher); - if (!guestToken) return { error: "fetch.fail" }; - - let tweet = await requestTweet(dispatcher, id, guestToken); - - // get new token & retry if old one expired - if ([403, 429].includes(tweet.status)) { - guestToken = await getGuestToken(dispatcher, true); - tweet = await requestTweet(dispatcher, id, guestToken) - } - - tweet = await tweet.json(); - +const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => { let tweetTypename = tweet?.data?.tweetResult?.result?.__typename; if (!tweetTypename) { @@ -127,13 +139,13 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { const reason = tweet?.data?.tweetResult?.result?.reason; switch(reason) { case "Protected": - return { error: "content.post.private" } + return { error: "content.post.private" }; case "NsfwLoggedOut": if (cookie) { tweet = await requestTweet(dispatcher, id, guestToken, cookie); tweet = await tweet.json(); tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - } else return { error: "content.post.age" } + } else return { error: "content.post.age" }; } } @@ -150,7 +162,69 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities; } - let media = (repostedTweet?.media || baseTweet?.extended_entities?.media); + return (repostedTweet?.media || baseTweet?.extended_entities?.media); +} + +const testResponse = (result) => { + const contentLength = result.headers.get("content-length"); + + if (!contentLength || contentLength === '0') { + return false; + } + + if (!result.headers.get("content-type").startsWith("application/json")) { + return false; + } + + return true; +} + +export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { + const cookie = await getCookie('twitter'); + + let syndication = false; + + let guestToken = await getGuestToken(dispatcher); + if (!guestToken) return { error: "fetch.fail" }; + + // for now we assume that graphql api will come back after some time, + // so we try it first + + let tweet = await requestTweet(dispatcher, id, guestToken); + + // get new token & retry if old one expired + if ([403, 429].includes(tweet.status)) { + guestToken = await getGuestToken(dispatcher, true); + if (cookie) { + tweet = await requestTweet(dispatcher, id, guestToken, cookie); + } else { + tweet = await requestTweet(dispatcher, id, guestToken); + } + } + + const testGraphql = testResponse(tweet); + + // if graphql requests fail, then resort to tweet embed api + if (!testGraphql) { + syndication = true; + tweet = await requestSyndication(dispatcher, id); + + const testSyndication = testResponse(tweet); + + // if even syndication request failed, then cry out loud + if (!testSyndication) { + return { error: "fetch.fail" }; + } + } + + tweet = await tweet.json(); + + let media = + syndication + ? tweet.mediaDetails + : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie); + + if (!media) return { error: "fetch.empty" }; // check if there's a video at given index (/video/) if (index >= 0 && index < media?.length) { @@ -163,7 +237,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { service: "twitter", type: "proxy", url, filename, - }) + }); switch (media?.length) { case undefined: diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index a4c54a8..e1cbf01 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -4,7 +4,8 @@ import { fetch } from "undici"; import { Innertube, Session } from "youtubei.js"; import { env } from "../../config.js"; -import { getCookie, updateCookieValues } from "../cookie/manager.js"; +import { getCookie } from "../cookie/manager.js"; +import { getYouTubeSession } from "../helpers/youtube-session.js"; const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms @@ -41,43 +42,30 @@ const hlsCodecList = { } } -const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320]; - -const transformSessionData = (cookie) => { - if (!cookie) - return; - - const values = { ...cookie.values() }; - const REQUIRED_VALUES = ['access_token', 'refresh_token']; - - if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) { - return; - } - - if (values.expires) { - values.expiry_date = values.expires; - delete values.expires; - } else if (!values.expiry_date) { - return; - } +const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDROID']; - return values; -} +const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320]; -const cloneInnertube = async (customFetch) => { +const cloneInnertube = async (customFetch, useSession) => { const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date(); const rawCookie = getCookie('youtube'); - const rawCookieValues = rawCookie?.values(); const cookie = rawCookie?.toString(); + const sessionTokens = getYouTubeSession(); + const retrieve_player = Boolean(sessionTokens || cookie); + + if (useSession && env.ytSessionServer && !sessionTokens?.potoken) { + throw "no_session_tokens"; + } + if (!innertube || shouldRefreshPlayer) { innertube = await Innertube.create({ fetch: customFetch, - retrieve_player: !!cookie, + retrieve_player, cookie, - po_token: rawCookieValues?.po_token, - visitor_data: rawCookieValues?.visitor_data, + po_token: useSession ? sessionTokens?.potoken : undefined, + visitor_data: useSession ? sessionTokens?.visitor_data : undefined, }); lastRefreshedAt = +new Date(); } @@ -93,73 +81,62 @@ const cloneInnertube = async (customFetch) => { innertube.session.cache ); - const oauthCookie = getCookie('youtube_oauth'); - const oauthData = transformSessionData(oauthCookie); + const yt = new Innertube(session); + return yt; +} - if (!session.logged_in && oauthData) { - await session.oauth.init(oauthData); - session.logged_in = true; - } +export default async function (o) { + const quality = o.quality === "max" ? 9000 : Number(o.quality); - if (session.logged_in && oauthData) { - if (session.oauth.shouldRefreshToken()) { - await session.oauth.refreshAccessToken(); - } + let useHLS = o.youtubeHLS; + let innertubeClient = o.innertubeClient || env.customInnertubeClient || "IOS"; - const cookieValues = oauthCookie.values(); - const oldExpiry = new Date(cookieValues.expiry_date); - const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date); + // HLS playlists from the iOS client don't contain the av1 video format. + if (useHLS && o.format === "av1") { + useHLS = false; + } - if (oldExpiry.getTime() !== newExpiry.getTime()) { - updateCookieValues(oauthCookie, { - ...session.oauth.client_id, - ...session.oauth.oauth2_tokens, - expiry_date: newExpiry.toISOString() - }); - } + if (useHLS) { + innertubeClient = "IOS"; } - const yt = new Innertube(session); - return yt; -} + // iOS client doesn't have adaptive formats of resolution >1080p, + // so we use the WEB_EMBEDDED client instead for those cases + const useSession = + env.ytSessionServer && ( + ( + !useHLS + && innertubeClient === "IOS" + && ( + (quality > 1080 && o.format !== "h264") + || (quality > 1080 && o.format !== "vp9") + ) + ) + ); + + if (useSession) { + innertubeClient = env.ytSessionInnertubeClient || "WEB_EMBEDDED"; + } -export default async function (o) { let yt; try { yt = await cloneInnertube( (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher - }) + }), + useSession ); } catch (e) { - if (e.message?.endsWith("decipher algorithm")) { + if (e === "no_session_tokens") { + return { error: "youtube.no_session_tokens" }; + } else if (e.message?.endsWith("decipher algorithm")) { return { error: "youtube.decipher" } } else if (e.message?.includes("refresh access token")) { return { error: "youtube.token_expired" } } else throw e; } - const cookie = getCookie('youtube')?.toString(); - - let useHLS = o.youtubeHLS; - - // HLS playlists don't contain the av1 video format, at least with the iOS client - if (useHLS && o.format === "av1") { - useHLS = false; - } - - let innertubeClient = o.innertubeClient || "ANDROID"; - - if (cookie) { - useHLS = false; - innertubeClient = "WEB"; - } - - if (useHLS) { - innertubeClient = "IOS"; - } - let info; try { info = await yt.getBasicInfo(o.id, innertubeClient); @@ -238,8 +215,6 @@ export default async function (o) { } } - const quality = o.quality === "max" ? 9000 : Number(o.quality); - const normalizeQuality = res => { const shortestSide = Math.min(res.height, res.width); return videoQualities.find(qual => qual >= shortestSide); @@ -428,6 +403,10 @@ export default async function (o) { } } + if (video?.drm_families || audio?.drm_families) { + return { error: "youtube.drm" }; + } + const fileMetadata = { title: basicInfo.title.trim(), artist: basicInfo.author.replace("- Topic", "").trim() @@ -474,7 +453,7 @@ export default async function (o) { urls = audio.uri; } - if (innertubeClient === "WEB" && innertube) { + if (!clientsWithNoCipher.includes(innertubeClient) && innertube) { urls = audio.decipher(innertube.session.player); } @@ -509,7 +488,7 @@ export default async function (o) { filenameAttributes.resolution = `${video.width}x${video.height}`; filenameAttributes.extension = codecList[codec].container; - if (innertubeClient === "WEB" && innertube) { + if (!clientsWithNoCipher.includes(innertubeClient) && innertube) { video = video.decipher(innertube.session.player); audio = audio.decipher(innertube.session.player); } else { diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json index 4fc5900..4139e39 100644 --- a/api/src/util/tests/twitter.json +++ b/api/src/util/tests/twitter.json @@ -169,6 +169,15 @@ "status": "tunnel" } }, + { + "name": "gif", + "url": "https://x.com/thelastromances/status/1897839691212202479", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "inexistent post", "url": "https://twitter.com/test/status/9487653", diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json index 0cca939..a169cc2 100644 --- a/api/src/util/tests/xiaohongshu.json +++ b/api/src/util/tests/xiaohongshu.json @@ -1,7 +1,8 @@ [ { - "name": "long link video", - "url": "https://www.xiaohongshu.com/discovery/item/6789065900000000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share", + "name": "video (might have expired)", + "url": "https://www.xiaohongshu.com/explore/67cc17a3000000000e00726a?xsec_token=CBSFRtbF57so920elY1kbIX4fE1nhrwlpGZs9m6pIFpwo=", + "canFail": true, "params": {}, "expected": { "code": 200, @@ -9,8 +10,9 @@ } }, { - "name": "picker with multiple live photos", - "url": "https://www.xiaohongshu.com/explore/67847fa1000000000203e6ed?xsec_token=CBzyP7Y44PPpsM20lgxqrIIJMHqOLemusDsRcmsX0cTpk", + "name": "picker with multiple live photos (might have expired)", + "url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=", + "canFail": true, "params": {}, "expected": { "code": 200, @@ -18,8 +20,9 @@ } }, { - "name": "one photo", + "name": "one photo (might have expired)", "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA", + "canFail": true, "params": {}, "expected": { "code": 200, @@ -27,7 +30,7 @@ } }, { - "name": "short link, might expire eventually", + "name": "short link (might have expired)", "url": "https://xhslink.com/a/czn4z6c1tic4", "canFail": true, "params": {}, diff --git a/api/src/util/tests/youtube.json b/api/src/util/tests/youtube.json index 0655e68..cb4964b 100644 --- a/api/src/util/tests/youtube.json +++ b/api/src/util/tests/youtube.json @@ -189,6 +189,7 @@ { "name": "hls video (h264, 1440p)", "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, "params": { "youtubeVideoCodec": "h264", "videoQuality": "1440", @@ -202,6 +203,7 @@ { "name": "hls video (vp9, 360p)", "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, "params": { "youtubeVideoCodec": "vp9", "videoQuality": "360", @@ -215,6 +217,7 @@ { "name": "hls video (audio mode)", "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, "params": { "downloadMode": "audio", "youtubeHLS": true @@ -227,6 +230,7 @@ { "name": "hls video (audio mode, best format)", "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, "params": { "downloadMode": "audio", "youtubeHLS": true,