소스 검색

Use env variable for prompts dir, refactor events

The prompts directory is now specified as an environment variable to be
more compatible with custom locations.

The events have also been refactored.
master
부모
커밋
796a8ff49b
6개의 변경된 파일89개의 추가작업 그리고 41개의 파일을 삭제
  1. +3
    -0
      .env.example
  2. +8
    -2
      src/common.ts
  3. +5
    -0
      src/config.ts
  4. +2
    -4
      src/index.ts
  5. +60
    -22
      src/summarizer.ts
  6. +11
    -13
      src/video-types/youtube/index.ts

+ 3
- 0
.env.example 파일 보기

@@ -3,3 +3,6 @@ OPENAI_API_KEY=

# OpenAI organization ID.
OPENAI_ORGANIZATION_ID=

# Directory where the prompts are stored.
OPENAI_PROMPTS_DIR=

+ 8
- 2
src/common.ts 파일 보기

@@ -17,10 +17,16 @@ export interface SummarizerEventEmitter extends NodeJS.EventEmitter {
on(eventType: 'end', callback: () => void): this;
}

export interface OpenAiParams {
apiKey: string;
organizationId?: string;
model?: string;
temperature?: number;
}

export interface CreateBaseSummarizerParams {
url: string;
language?: string;
country?: string;
openaiApiKey: string;
openaiOrganizationId?: string;
openAiParams: OpenAiParams;
}

+ 5
- 0
src/config.ts 파일 보기

@@ -0,0 +1,5 @@
export namespace openAi {
export const apiKey = process.env.OPENAI_API_KEY as string;
export const organizationId = process.env.OPENAI_ORGANIZATION_ID;
export const promptsDir = process.env.OPENAI_PROMPTS_DIR as string;
}

+ 2
- 4
src/index.ts 파일 보기

@@ -16,8 +16,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven
const {
type: videoType,
url,
openaiOrganizationId,
openaiApiKey,
openAiParams,
language,
country,
} = params;
@@ -26,8 +25,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven
case VideoType.YOUTUBE:
return new YouTubeSummarizerEventEmitter({
url,
openaiOrganizationId,
openaiApiKey,
openAiParams,
language,
country,
});


+ 60
- 22
src/summarizer.ts 파일 보기

@@ -2,8 +2,32 @@ import fetchPonyfill from 'fetch-ponyfill';
import Handlebars from 'handlebars';
import { resolve } from 'path';
import { readFile } from 'fs/promises';
import * as config from './config';
import { OpenAiParams } from './common';

export interface MakeAiCallParams {
prompts: string[];
openAiParams: OpenAiParams;
}

export class AiCallError extends Error {
constructor(message: string, public readonly response: Response) {
super(message);
this.name = 'AiCallError';
}
}

const makeAiCall = async (params: MakeAiCallParams): Promise<string> => {
const {
prompts,
openAiParams: {
apiKey,
organizationId,
model = 'gpt-3.5-turbo',
temperature = 0.6,
},
} = params;

const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: string): Promise<string> => {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
Accept: 'application/json',
@@ -21,9 +45,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
method: 'POST',
headers,
body: JSON.stringify({
//model: 'gpt-4',
model: 'gpt-3.5-turbo',
temperature: 0.6,
model,
temperature,
messages: [
{
role: 'user',
@@ -35,9 +58,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
);

if (!response.ok) {
const responseText = await response.text();
console.log(responseText);
throw new Error(`OpenAI API call failed with status ${response.status}`);
const { error } = await response.json();
throw new AiCallError(`OpenAI API call failed with status ${response.status}: ${error.message}`, response);
}

const { choices } = await response.json();
@@ -47,38 +69,54 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
};

const compilePrompts = async (filename: string, params: Record<string, unknown>): Promise<string[]> => {
const rawPromptText = await readFile(resolve(__dirname, filename), 'utf-8');
const rawPromptText = await readFile(resolve(config.openAi.promptsDir, filename), 'utf-8');
const fill = Handlebars.compile(rawPromptText, { noEscape: true });
const filledText = fill(params);
return filledText.split('---').map((s) => s.trim());
};

export const normalizeTranscriptText = async (
export interface NormalizeTranscriptTextParams {
rawTranscriptText: string,
apiKey: string,
organizationId?: string,
) => {
openAiParams: OpenAiParams,
}

export const normalizeTranscriptText = async (params: NormalizeTranscriptTextParams) => {
const {
rawTranscriptText,
openAiParams,
} = params;
const prompts = await compilePrompts(
'../prompts/normalize-transcript-text.hbs',
'normalize-transcript-text.hbs',
{
transcript: rawTranscriptText,
},
);

return makeAiCall(prompts, apiKey, organizationId);
return makeAiCall({
prompts,
openAiParams,
});
};

export const summarizeTranscript = async (
transcript: string,
apiKey: string,
organizationId?: string,
) => {
export interface SummarizeTranscriptParams {
normalizedTranscript: string,
openAiParams: OpenAiParams,
}

export const summarizeTranscript = async (params: SummarizeTranscriptParams) => {
const {
normalizedTranscript,
openAiParams,
} = params;
const prompts = await compilePrompts(
'../prompts/summarize-transcript.hbs',
'summarize-transcript.hbs',
{
transcript,
transcript: normalizedTranscript,
},
);

return makeAiCall(prompts, apiKey, organizationId);
return makeAiCall({
prompts,
openAiParams,
});
};

+ 11
- 13
src/video-types/youtube/index.ts 파일 보기

@@ -8,7 +8,7 @@ import {
} from './transcript';
import { normalizeTranscriptText, summarizeTranscript } from '../../summarizer';

export interface CreateYouTubeSummarizerParams extends CreateBaseSummarizerParams {}
export type CreateYouTubeSummarizerParams = CreateBaseSummarizerParams

export class YouTubeSummarizerEventEmitter extends EventEmitter implements SummarizerEventEmitter {
constructor(private readonly params: CreateYouTubeSummarizerParams) {
@@ -18,8 +18,7 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
process() {
const {
url,
openaiApiKey,
openaiOrganizationId,
openAiParams,
...config
} = this.params;
const identifier = retrieveVideoId(url);
@@ -52,21 +51,20 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
});

this.emit('process', {
processType: 'normalize-caption',
processType: 'normalize-transcript',
phase: 'start',
});

return normalizeTranscriptText(
transcript.map((item) => item.text).join(' '),
openaiApiKey,
openaiOrganizationId,
);
return normalizeTranscriptText({
rawTranscriptText: transcript.map((item) => item.text).join(' '),
openAiParams,
});
})
.then((normalizedCaption) => {
.then((normalizedTranscript) => {
this.emit('process', {
processType: 'normalize-transcript',
phase: 'success',
content: normalizedCaption,
content: normalizedTranscript,
contentType: 'text/plain',
});

@@ -75,13 +73,13 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
phase: 'start',
});

return summarizeTranscript(normalizedCaption, openaiApiKey, openaiOrganizationId);
return summarizeTranscript({ normalizedTranscript, openAiParams });
})
.then((summary) => {
this.emit('process', {
processType: 'summarize-transcript',
phase: 'success',
data: summary,
content: summary,
contentType: 'text/plain',
});



불러오는 중...
취소
저장