|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223 |
- // based from https://github.com/Kakulukian/youtube-transcript
- //
- // we refactored it to make it more maintainable
-
- import fetchPonyfill from 'fetch-ponyfill';
- import {
- CannotRetrieveVideoPageError,
- FetchTranscriptRequestFailureError,
- InnerTubeApiKeyMissingError,
- InvalidTranscriptActionsError,
- InvalidTranscriptResponseContextError,
- } from './errors';
- import { BaseTranscriptItem } from '../../transcript';
- import { generateNonce } from './crypto';
- import { retrieveVideoId } from './url';
- import { VIDEO_TYPE } from './common';
-
- export interface CreateTranscriptFetcherParams {
- type: typeof VIDEO_TYPE;
- }
-
- interface TranscriptConfig {
- language?: string;
- country?: string;
- }
-
- export interface TranscriptItem extends BaseTranscriptItem {
- duration: number;
- offset: number;
- }
-
- const { fetch: f } = fetchPonyfill();
-
- const getVideoPage = async (videoId: string): Promise<string> => {
- const identifier = retrieveVideoId(videoId);
- const videoUrl = new URL('/watch', 'https://www.youtube.com');
- const videoUrlParams = new URLSearchParams({
- v: identifier,
- });
- videoUrl.search = videoUrlParams.toString();
- const videoPageResponse = await f(videoUrl.toString());
- if (videoPageResponse.ok) {
- return videoPageResponse.text();
- }
- throw new CannotRetrieveVideoPageError('Unable to get video page.');
- };
-
- const extractInnerTubeApiKeyFromPage = (videoPageBody: string): string => videoPageBody
- .split('"INNERTUBE_API_KEY":"')[1]
- .split('"')[0];
-
- const extractSerializedShareEntityFromPage = (page: string) => page.split('"serializedShareEntity":"')[1]?.split('"')[0];
-
- const extractVisitorDataFromPage = (page: string) => page.split('"VISITOR_DATA":"')[1]?.split('"')[0];
-
- const extractSessionIdFromPage = (page: string) => page.split('"sessionId":"')[1]?.split('"')[0];
-
- const extractClickTrackingParamsFromPage = (page: string) => page
- ?.split('"clickTrackingParams":"')[1]
- ?.split('"')[0];
-
- interface VideoPageData {
- innerTubeApiKey?: string;
- serializedShareEntity?: string;
- visitorData?: string;
- sessionId?: string;
- clickTrackingParams?: string;
- }
-
- interface TranscriptResponse {
- responseContext?: unknown,
- actions?: {
- updateEngagementPanelAction: {
- content: {
- transcriptRenderer: {
- body: {
- transcriptBodyRenderer: {
- cueGroups: Cue[],
- }
- }
- }
- }
- },
- }[];
- }
-
- interface Cue {
- transcriptCueGroupRenderer: {
- cues: {
- transcriptCueRenderer: {
- cue: {
- simpleText: string;
- },
- durationMs: string;
- startOffsetMs: string;
- }
- }[],
- },
- }
-
- const extractDataFromPage = (page: string): VideoPageData => ({
- innerTubeApiKey: extractInnerTubeApiKeyFromPage(page),
- serializedShareEntity: extractSerializedShareEntityFromPage(page),
- visitorData: extractVisitorDataFromPage(page),
- sessionId: extractSessionIdFromPage(page),
- clickTrackingParams: extractClickTrackingParamsFromPage(page),
- });
-
- const generateGetTranscriptRequestBody = (
- p: Partial<VideoPageData>,
- config?: TranscriptConfig,
- ) => {
- const {
- serializedShareEntity,
- visitorData,
- sessionId,
- clickTrackingParams,
- } = p;
- return {
- context: {
- client: {
- hl: config?.language ?? 'en',
- gl: config?.country ?? 'PH',
- visitorData,
- userAgent:
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)',
- clientName: 'WEB',
- clientVersion: '2.20200925.01.00',
- osName: 'Macintosh',
- osVersion: '10_15_4',
- browserName: 'Chrome',
- browserVersion: '85.0f.4183.83',
- screenWidthPoints: 1440,
- screenHeightPoints: 770,
- screenPixelDensity: 2,
- utcOffsetMinutes: 120,
- userInterfaceTheme: 'USER_INTERFACE_THEME_LIGHT',
- connectionType: 'CONN_CELLULAR_3G',
- },
- request: {
- sessionId,
- internalExperimentFlags: [],
- consistencyTokenJars: [],
- },
- user: {},
- clientScreenNonce: generateNonce(),
- clickTracking: {
- clickTrackingParams,
- },
- },
- params: serializedShareEntity,
- };
- };
-
- const fetchTranscriptItems = async (pageData: VideoPageData, config?: TranscriptConfig) => {
- const { innerTubeApiKey } = pageData;
- if (!(innerTubeApiKey && innerTubeApiKey.length > 0)) {
- throw new InnerTubeApiKeyMissingError('InnerTube API key not found on video page.');
- }
- const getTranscriptUrl = new URL('/youtubei/v1/get_transcript', 'https://www.youtube.com');
- const getTranscriptParams = new URLSearchParams({
- key: innerTubeApiKey,
- });
- getTranscriptUrl.search = getTranscriptParams.toString();
- const transcriptResponse = await f(getTranscriptUrl.toString(), {
- method: 'POST',
- headers: {
- Accept: 'application/json',
- 'Content-Type': 'application/json',
- },
- body: JSON.stringify(generateGetTranscriptRequestBody(pageData, config)),
- });
-
- if (!transcriptResponse.ok) {
- throw new FetchTranscriptRequestFailureError(`Fetching transcript failed with status ${transcriptResponse.status}.`);
- }
-
- const transcriptBody = await transcriptResponse.json() as TranscriptResponse;
- if (!transcriptBody.responseContext) {
- throw new InvalidTranscriptResponseContextError('No responseContext found on get transcript response.');
- }
-
- if (!transcriptBody.actions) {
- throw new InvalidTranscriptActionsError('No actions found on get transcript response.');
- }
-
- const { cueGroups: transcripts } = transcriptBody
- .actions[0]
- .updateEngagementPanelAction
- .content
- .transcriptRenderer
- .body
- .transcriptBodyRenderer;
-
- return transcripts.map((cue: Cue) => ({
- text: cue.transcriptCueGroupRenderer.cues[0].transcriptCueRenderer
- .cue.simpleText,
- duration: parseInt(
- cue.transcriptCueGroupRenderer.cues[0].transcriptCueRenderer
- .durationMs,
- 10,
- ),
- offset: parseInt(
- cue.transcriptCueGroupRenderer.cues[0].transcriptCueRenderer
- .startOffsetMs,
- 10,
- ),
- })) as TranscriptItem[];
- };
-
- export interface SummarizerProcessParams {
- url: string;
- language?: string;
- country?: string;
- }
-
- export const getRawTranscript = async (params: SummarizerProcessParams) => {
- const { url, ...config } = params;
- const identifier = retrieveVideoId(url);
- const videoPageBody = await getVideoPage(identifier);
- const pageData = extractDataFromPage(videoPageBody);
- return fetchTranscriptItems(pageData, config);
- };
|