webvideo-transcript-summary-core/src/video-types/youtube/transcript.ts

// based from https://github.com/Kakulukian/youtube-transcript
//
// we refactored it to make it more maintainable

import fetchPonyfill from 'fetch-ponyfill';
import {
  CannotRetrieveVideoPageError,
  FetchTranscriptRequestFailureError,
  InnerTubeApiKeyMissingError,
  InvalidTranscriptActionsError,
  InvalidTranscriptResponseContextError,
} from './errors';
import { BaseTranscriptItem } from '../../transcript';
import { generateNonce } from './crypto';
import { retrieveVideoId } from './url';
import { VIDEO_TYPE } from './common';

export interface CreateTranscriptFetcherParams {
  type: typeof VIDEO_TYPE;
}

interface TranscriptConfig {
  language?: string;
  country?: string;
}

export interface TranscriptItem extends BaseTranscriptItem {
  duration: number;
  offset: number;
}

const { fetch: f } = fetchPonyfill();

const getVideoPage = async (videoId: string): Promise<string> => {
  const identifier = retrieveVideoId(videoId);
  const videoUrl = new URL('/watch', 'https://www.youtube.com');
  const videoUrlParams = new URLSearchParams({
    v: identifier,
  });
  videoUrl.search = videoUrlParams.toString();
  const videoPageResponse = await f(videoUrl.toString());
  if (videoPageResponse.ok) {
    return videoPageResponse.text();
  }
  throw new CannotRetrieveVideoPageError('Unable to get video page.');
};

const extractInnerTubeApiKeyFromPage = (videoPageBody: string): string => videoPageBody
  .split('"INNERTUBE_API_KEY":"')[1]
  .split('"')[0];

const extractSerializedShareEntityFromPage = (page: string) => page.split('"serializedShareEntity":"')[1]?.split('"')[0];

const extractVisitorDataFromPage = (page: string) => page.split('"VISITOR_DATA":"')[1]?.split('"')[0];

const extractSessionIdFromPage = (page: string) => page.split('"sessionId":"')[1]?.split('"')[0];

const extractClickTrackingParamsFromPage = (page: string) => page
  ?.split('"clickTrackingParams":"')[1]
  ?.split('"')[0];

interface VideoPageData {
  innerTubeApiKey?: string;
  serializedShareEntity?: string;
  visitorData?: string;
  sessionId?: string;
  clickTrackingParams?: string;
}

interface TranscriptResponse {
  responseContext?: unknown,
  actions?: {
    updateEngagementPanelAction: {
      content: {
        transcriptRenderer: {
          body: {
            transcriptBodyRenderer: {
              cueGroups: Cue[],
            }
          }
        }
      }
    },
  }[];
}

interface Cue {
  transcriptCueGroupRenderer: {
    cues: {
      transcriptCueRenderer: {
        cue: {
          simpleText: string;
        },
        durationMs: string;
        startOffsetMs: string;
      }
    }[],
  },
}

const extractDataFromPage = (page: string): VideoPageData => ({
  innerTubeApiKey: extractInnerTubeApiKeyFromPage(page),
  serializedShareEntity: extractSerializedShareEntityFromPage(page),
  visitorData: extractVisitorDataFromPage(page),
  sessionId: extractSessionIdFromPage(page),
  clickTrackingParams: extractClickTrackingParamsFromPage(page),
});

const generateGetTranscriptRequestBody = (
  p: Partial<VideoPageData>,
  config?: TranscriptConfig,
) => {
  const {
    serializedShareEntity,
    visitorData,
    sessionId,
    clickTrackingParams,
  } = p;
  return {
    context: {
      client: {
        hl: config?.language ?? 'en',
        gl: config?.country ?? 'PH',
        visitorData,
        userAgent:
          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)',
        clientName: 'WEB',
        clientVersion: '2.20200925.01.00',
        osName: 'Macintosh',
        osVersion: '10_15_4',
        browserName: 'Chrome',
        browserVersion: '85.0f.4183.83',
        screenWidthPoints: 1440,
        screenHeightPoints: 770,
        screenPixelDensity: 2,
        utcOffsetMinutes: 120,
        userInterfaceTheme: 'USER_INTERFACE_THEME_LIGHT',
        connectionType: 'CONN_CELLULAR_3G',
      },
      request: {
        sessionId,
        internalExperimentFlags: [],
        consistencyTokenJars: [],
      },
      user: {},
      clientScreenNonce: generateNonce(),
      clickTracking: {
        clickTrackingParams,
      },
    },
    params: serializedShareEntity,
  };
};

const fetchTranscriptItems = async (pageData: VideoPageData, config?: TranscriptConfig) => {
  const { innerTubeApiKey } = pageData;
  if (!(innerTubeApiKey && innerTubeApiKey.length > 0)) {
    throw new InnerTubeApiKeyMissingError('InnerTube API key not found on video page.');
  }
  const getTranscriptUrl = new URL('/youtubei/v1/get_transcript', 'https://www.youtube.com');
  const getTranscriptParams = new URLSearchParams({
    key: innerTubeApiKey,
  });
  getTranscriptUrl.search = getTranscriptParams.toString();
  const transcriptResponse = await f(getTranscriptUrl.toString(), {
    method: 'POST',
    headers: {
      Accept: 'application/json',
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(generateGetTranscriptRequestBody(pageData, config)),
  });

  if (!transcriptResponse.ok) {
    throw new FetchTranscriptRequestFailureError(`Fetching transcript failed with status ${transcriptResponse.status}.`);
  }

  const transcriptBody = await transcriptResponse.json() as TranscriptResponse;
  if (!transcriptBody.responseContext) {
    throw new InvalidTranscriptResponseContextError('No responseContext found on get transcript response.');
  }

  if (!transcriptBody.actions) {
    throw new InvalidTranscriptActionsError('No actions found on get transcript response.');
  }

  const { cueGroups: transcripts } = transcriptBody
    .actions[0]
    .updateEngagementPanelAction
    .content
    .transcriptRenderer
    .body
    .transcriptBodyRenderer;

  return transcripts.map((cue: Cue) => ({
    text: cue.transcriptCueGroupRenderer.cues[0].transcriptCueRenderer
      .cue.simpleText,
    duration: parseInt(
      cue.transcriptCueGroupRenderer.cues[0].transcriptCueRenderer
        .durationMs,
      10,
    ),
    offset: parseInt(
      cue.transcriptCueGroupRenderer.cues[0].transcriptCueRenderer
        .startOffsetMs,
      10,
    ),
  })) as TranscriptItem[];
};

export interface SummarizerProcessParams {
  url: string;
  language?: string;
  country?: string;
}

export const getRawTranscript = async (params: SummarizerProcessParams) => {
  const { url, ...config } = params;
  const identifier = retrieveVideoId(url);
  const videoPageBody = await getVideoPage(identifier);
  const pageData = extractDataFromPage(videoPageBody);
  return fetchTranscriptItems(pageData, config);
};