Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Sync Upstream API (8d3db90)
  • Loading branch information
Sticks committed Jan 21, 2025
1 parent f681d64 commit ae221d2
Show file tree
Hide file tree
Showing 18 changed files with 418 additions and 52 deletions.
4 changes: 2 additions & 2 deletions api/package.json
@@ -1,7 +1,7 @@
{
"name": "@imput/cobalt-api",
"description": "save what you love",
"version": "10.5.4",
"version": "10.6",
"author": "imput",
"exports": "./src/cobalt.js",
"type": "module",
Expand Down Expand Up @@ -39,7 +39,7 @@
"set-cookie-parser": "2.6.0",
"undici": "^5.19.1",
"url-pattern": "1.0.3",
"youtubei.js": "^12.2.0",
"youtubei.js": "^13.0.0",
"zod": "^3.23.8"
},
"optionalDependencies": {
Expand Down
2 changes: 1 addition & 1 deletion api/src/core/api.js
Expand Up @@ -313,7 +313,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
...Object.entries(req.headers)
]);

return stream(res, { type: 'internal', ...streamInfo });
return stream(res, { type: 'internal', data: streamInfo });
};

app.get('/itunnel', itunnelHandler);
Expand Down
18 changes: 15 additions & 3 deletions api/src/misc/utils.js
@@ -1,8 +1,16 @@
export function getRedirectingURL(url) {
return fetch(url, { redirect: 'manual' }).then((r) => {
if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
const redirectStatuses = new Set([301, 302, 303, 307, 308]);

export async function getRedirectingURL(url, dispatcher) {
const location = await fetch(url, {
redirect: 'manual',
dispatcher,
}).then((r) => {
if (redirectStatuses.has(r.status) && r.headers.has('location')) {
return r.headers.get('location');
}
}).catch(() => null);

return location;
}

export function merge(a, b) {
Expand All @@ -29,3 +37,7 @@ export function splitFilenameExtension(filename) {
return [ parts.join('.'), ext ]
}
}

export function zip(a, b) {
return a.map((value, i) => [ value, b[i] ]);
}
7 changes: 5 additions & 2 deletions api/src/processing/match-action.js
Expand Up @@ -15,7 +15,8 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
filename: r.filenameAttributes ?
createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename,
fileMetadata: !disableMetadata ? r.fileMetadata : false,
requestIP
requestIP,
originalRequest: r.originalRequest
},
params = {};

Expand Down Expand Up @@ -47,7 +48,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
});

case "photo":
responseType = "redirect";
params = { type: "proxy" };
break;

case "gif":
Expand Down Expand Up @@ -83,6 +84,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
case "twitter":
case "snapchat":
case "bsky":
case "xiaohongshu":
params = { picker: r.picker };
break;

Expand Down Expand Up @@ -143,6 +145,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
case "ok":
case "vk":
case "tiktok":
case "xiaohongshu":
params = { type: "proxy" };
break;

Expand Down
10 changes: 10 additions & 0 deletions api/src/processing/match.js
Expand Up @@ -28,6 +28,7 @@ import snapchat from "./services/snapchat.js";
import loom from "./services/loom.js";
import facebook from "./services/facebook.js";
import bluesky from "./services/bluesky.js";
import xiaohongshu from "./services/xiaohongshu.js";

let freebind;

Expand Down Expand Up @@ -239,6 +240,15 @@ export default async function({ host, patternMatch, params }) {
});
break;

case "xiaohongshu":
r = await xiaohongshu({
...patternMatch,
h265: params.tiktokH265,
isAudioOnly,
dispatcher,
});
break;

default:
return createResponse("error", {
code: "error.api.service.unsupported"
Expand Down
8 changes: 8 additions & 0 deletions api/src/processing/service-config.js
Expand Up @@ -166,6 +166,14 @@ export const services = {
subdomains: ["m"],
altDomains: ["vkvideo.ru", "vk.ru"],
},
xiaohongshu: {
patterns: [
"explore/:id?xsec_token=:token",
"discovery/item/:id?xsec_token=:token",
"a/:shareId"
],
altDomains: ["xhslink.com"],
},
youtube: {
patterns: [
"watch?v=:id",
Expand Down
4 changes: 4 additions & 0 deletions api/src/processing/service-patterns.js
Expand Up @@ -71,4 +71,8 @@ export const testers = {

"bsky": pattern =>
pattern.user?.length <= 128 && pattern.post?.length <= 128,

"xiaohongshu": pattern =>
pattern.id?.length <= 24 && pattern.token?.length <= 64
|| pattern.shareId?.length <= 12,
}
65 changes: 49 additions & 16 deletions api/src/processing/services/bluesky.js
Expand Up @@ -71,6 +71,24 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
return { picker };
}

const extractGif = ({ url, filename }) => {
const gifUrl = new URL(url);

if (!gifUrl || gifUrl.hostname !== "media.tenor.com") {
return { error: "fetch.empty" };
}

// remove downscaling params from gif url
// such as "?hh=498&ww=498"
gifUrl.search = "";

return {
urls: gifUrl,
isPhoto: true,
filename: `${filename}.gif`,
}
}

export default async function ({ user, post, alwaysProxy, dispatcher }) {
const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0");
apiEndpoint.searchParams.set(
Expand Down Expand Up @@ -102,22 +120,37 @@ export default async function ({ user, post, alwaysProxy, dispatcher }) {
const embedType = getPost?.thread?.post?.embed?.$type;
const filename = `bluesky_${user}_${post}`;

if (embedType === "app.bsky.embed.video#view") {
return extractVideo({
media: getPost.thread?.post?.embed,
filename,
})
}

if (embedType === "app.bsky.embed.recordWithMedia#view") {
return extractVideo({
media: getPost.thread?.post?.embed?.media,
filename,
})
}

if (embedType === "app.bsky.embed.images#view") {
return extractImages({ getPost, filename, alwaysProxy });
switch (embedType) {
case "app.bsky.embed.video#view":
return extractVideo({
media: getPost.thread?.post?.embed,
filename,
});

case "app.bsky.embed.images#view":
return extractImages({
getPost,
filename,
alwaysProxy
});

case "app.bsky.embed.external#view":
return extractGif({
url: getPost?.thread?.post?.embed?.external?.uri,
filename,
});

case "app.bsky.embed.recordWithMedia#view":
if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") {
return extractGif({
url: getPost?.thread?.post?.embed?.media?.external?.uri,
filename,
});
}
return extractVideo({
media: getPost.thread?.post?.embed?.media,
filename,
});
}

return { error: "fetch.empty" };
Expand Down
2 changes: 1 addition & 1 deletion api/src/processing/services/tiktok.js
Expand Up @@ -30,7 +30,7 @@ export default async function(obj) {
if (!postId) return { error: "fetch.short_link" };

// should always be /video/, even for photos
const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
const res = await fetch(`https://www.tiktok.com/@i/video/${postId}`, {
headers: {
"user-agent": genericUserAgent,
cookie,
Expand Down
116 changes: 116 additions & 0 deletions api/src/processing/services/xiaohongshu.js
@@ -0,0 +1,116 @@
import { extract, normalizeURL } from "../url.js";
import { genericUserAgent } from "../../config.js";
import { createStream } from "../../stream/manage.js";
import { getRedirectingURL } from "../../misc/utils.js";

const https = (url) => {
return url.replace(/^http:/i, 'https:');
}

export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) {
let noteId = id;
let xsecToken = token;

if (!noteId) {
const extractedURL = await getRedirectingURL(
`https://xhslink.com/a/${shareId}`,
dispatcher
);

if (extractedURL) {
const { patternMatch } = extract(normalizeURL(extractedURL));

if (patternMatch) {
noteId = patternMatch.id;
xsecToken = patternMatch.token;
}
}
}

if (!noteId || !xsecToken) return { error: "fetch.short_link" };

const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, {
headers: {
"user-agent": genericUserAgent,
},
dispatcher,
});

const html = await res.text();

let note;
try {
const initialState = html
.split('<script>window.__INITIAL_STATE__=')[1]
.split('</script>')[0]
.replace(/:\s*undefined/g, ":null");

const data = JSON.parse(initialState);

const noteInfo = data?.note?.noteDetailMap;
if (!noteInfo) throw "no note detail map";

const currentNote = noteInfo[noteId];
if (!currentNote) throw "no current note in detail map";

note = currentNote.note;
} catch {}

if (!note) return { error: "fetch.empty" };

const video = note.video;
const images = note.imageList;

const filenameBase = `xiaohongshu_${noteId}`;

if (video) {
const videoFilename = `${filenameBase}.mp4`;
const audioFilename = `${filenameBase}_audio`;

let videoURL;

if (h265 && !isAudioOnly && video.consumer?.originVideoKey) {
videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`;
} else {
const h264Streams = video.media?.stream?.h264;

if (h264Streams?.length) {
videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl;
}
}

if (!videoURL) return { error: "fetch.empty" };

return {
urls: https(videoURL),
filename: videoFilename,
audioFilename: audioFilename,
}
}

if (!images || images.length === 0) {
return { error: "fetch.empty" };
}

if (images.length === 1) {
return {
isPhoto: true,
urls: https(images[0].urlDefault),
filename: `${filenameBase}.jpg`,
}
}

const picker = images.map((image, i) => {
return {
type: "photo",
url: createStream({
service: "xiaohongshu",
type: "proxy",
url: https(image.urlDefault),
filename: `${filenameBase}_${i + 1}.jpg`,
})
}
});

return { picker };
}

0 comments on commit ae221d2

Please sign in to comment.