The prompts directory is now specified as an environment variable to be more compatible with custom locations. The events have also been refactored.master
@@ -3,3 +3,6 @@ OPENAI_API_KEY= | |||
# OpenAI organization ID. | |||
OPENAI_ORGANIZATION_ID= | |||
# Directory where the prompts are stored. | |||
OPENAI_PROMPTS_DIR= |
@@ -17,10 +17,16 @@ export interface SummarizerEventEmitter extends NodeJS.EventEmitter { | |||
on(eventType: 'end', callback: () => void): this; | |||
} | |||
export interface OpenAiParams { | |||
apiKey: string; | |||
organizationId?: string; | |||
model?: string; | |||
temperature?: number; | |||
} | |||
export interface CreateBaseSummarizerParams { | |||
url: string; | |||
language?: string; | |||
country?: string; | |||
openaiApiKey: string; | |||
openaiOrganizationId?: string; | |||
openAiParams: OpenAiParams; | |||
} |
@@ -0,0 +1,5 @@ | |||
export namespace openAi { | |||
export const apiKey = process.env.OPENAI_API_KEY as string; | |||
export const organizationId = process.env.OPENAI_ORGANIZATION_ID; | |||
export const promptsDir = process.env.OPENAI_PROMPTS_DIR as string; | |||
} |
@@ -16,8 +16,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven | |||
const { | |||
type: videoType, | |||
url, | |||
openaiOrganizationId, | |||
openaiApiKey, | |||
openAiParams, | |||
language, | |||
country, | |||
} = params; | |||
@@ -26,8 +25,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven | |||
case VideoType.YOUTUBE: | |||
return new YouTubeSummarizerEventEmitter({ | |||
url, | |||
openaiOrganizationId, | |||
openaiApiKey, | |||
openAiParams, | |||
language, | |||
country, | |||
}); | |||
@@ -2,8 +2,32 @@ import fetchPonyfill from 'fetch-ponyfill'; | |||
import Handlebars from 'handlebars'; | |||
import { resolve } from 'path'; | |||
import { readFile } from 'fs/promises'; | |||
import * as config from './config'; | |||
import { OpenAiParams } from './common'; | |||
export interface MakeAiCallParams { | |||
prompts: string[]; | |||
openAiParams: OpenAiParams; | |||
} | |||
export class AiCallError extends Error { | |||
constructor(message: string, public readonly response: Response) { | |||
super(message); | |||
this.name = 'AiCallError'; | |||
} | |||
} | |||
const makeAiCall = async (params: MakeAiCallParams): Promise<string> => { | |||
const { | |||
prompts, | |||
openAiParams: { | |||
apiKey, | |||
organizationId, | |||
model = 'gpt-3.5-turbo', | |||
temperature = 0.6, | |||
}, | |||
} = params; | |||
const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: string): Promise<string> => { | |||
const headers: Record<string, string> = { | |||
'Content-Type': 'application/json', | |||
Accept: 'application/json', | |||
@@ -21,9 +45,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st | |||
method: 'POST', | |||
headers, | |||
body: JSON.stringify({ | |||
//model: 'gpt-4', | |||
model: 'gpt-3.5-turbo', | |||
temperature: 0.6, | |||
model, | |||
temperature, | |||
messages: [ | |||
{ | |||
role: 'user', | |||
@@ -35,9 +58,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st | |||
); | |||
if (!response.ok) { | |||
const responseText = await response.text(); | |||
console.log(responseText); | |||
throw new Error(`OpenAI API call failed with status ${response.status}`); | |||
const { error } = await response.json(); | |||
throw new AiCallError(`OpenAI API call failed with status ${response.status}: ${error.message}`, response); | |||
} | |||
const { choices } = await response.json(); | |||
@@ -47,38 +69,54 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st | |||
}; | |||
const compilePrompts = async (filename: string, params: Record<string, unknown>): Promise<string[]> => { | |||
const rawPromptText = await readFile(resolve(__dirname, filename), 'utf-8'); | |||
const rawPromptText = await readFile(resolve(config.openAi.promptsDir, filename), 'utf-8'); | |||
const fill = Handlebars.compile(rawPromptText, { noEscape: true }); | |||
const filledText = fill(params); | |||
return filledText.split('---').map((s) => s.trim()); | |||
}; | |||
export const normalizeTranscriptText = async ( | |||
export interface NormalizeTranscriptTextParams { | |||
rawTranscriptText: string, | |||
apiKey: string, | |||
organizationId?: string, | |||
) => { | |||
openAiParams: OpenAiParams, | |||
} | |||
export const normalizeTranscriptText = async (params: NormalizeTranscriptTextParams) => { | |||
const { | |||
rawTranscriptText, | |||
openAiParams, | |||
} = params; | |||
const prompts = await compilePrompts( | |||
'../prompts/normalize-transcript-text.hbs', | |||
'normalize-transcript-text.hbs', | |||
{ | |||
transcript: rawTranscriptText, | |||
}, | |||
); | |||
return makeAiCall(prompts, apiKey, organizationId); | |||
return makeAiCall({ | |||
prompts, | |||
openAiParams, | |||
}); | |||
}; | |||
export const summarizeTranscript = async ( | |||
transcript: string, | |||
apiKey: string, | |||
organizationId?: string, | |||
) => { | |||
export interface SummarizeTranscriptParams { | |||
normalizedTranscript: string, | |||
openAiParams: OpenAiParams, | |||
} | |||
export const summarizeTranscript = async (params: SummarizeTranscriptParams) => { | |||
const { | |||
normalizedTranscript, | |||
openAiParams, | |||
} = params; | |||
const prompts = await compilePrompts( | |||
'../prompts/summarize-transcript.hbs', | |||
'summarize-transcript.hbs', | |||
{ | |||
transcript, | |||
transcript: normalizedTranscript, | |||
}, | |||
); | |||
return makeAiCall(prompts, apiKey, organizationId); | |||
return makeAiCall({ | |||
prompts, | |||
openAiParams, | |||
}); | |||
}; |
@@ -8,7 +8,7 @@ import { | |||
} from './transcript'; | |||
import { normalizeTranscriptText, summarizeTranscript } from '../../summarizer'; | |||
export interface CreateYouTubeSummarizerParams extends CreateBaseSummarizerParams {} | |||
export type CreateYouTubeSummarizerParams = CreateBaseSummarizerParams | |||
export class YouTubeSummarizerEventEmitter extends EventEmitter implements SummarizerEventEmitter { | |||
constructor(private readonly params: CreateYouTubeSummarizerParams) { | |||
@@ -18,8 +18,7 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa | |||
process() { | |||
const { | |||
url, | |||
openaiApiKey, | |||
openaiOrganizationId, | |||
openAiParams, | |||
...config | |||
} = this.params; | |||
const identifier = retrieveVideoId(url); | |||
@@ -52,21 +51,20 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa | |||
}); | |||
this.emit('process', { | |||
processType: 'normalize-caption', | |||
processType: 'normalize-transcript', | |||
phase: 'start', | |||
}); | |||
return normalizeTranscriptText( | |||
transcript.map((item) => item.text).join(' '), | |||
openaiApiKey, | |||
openaiOrganizationId, | |||
); | |||
return normalizeTranscriptText({ | |||
rawTranscriptText: transcript.map((item) => item.text).join(' '), | |||
openAiParams, | |||
}); | |||
}) | |||
.then((normalizedCaption) => { | |||
.then((normalizedTranscript) => { | |||
this.emit('process', { | |||
processType: 'normalize-transcript', | |||
phase: 'success', | |||
content: normalizedCaption, | |||
content: normalizedTranscript, | |||
contentType: 'text/plain', | |||
}); | |||
@@ -75,13 +73,13 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa | |||
phase: 'start', | |||
}); | |||
return summarizeTranscript(normalizedCaption, openaiApiKey, openaiOrganizationId); | |||
return summarizeTranscript({ normalizedTranscript, openAiParams }); | |||
}) | |||
.then((summary) => { | |||
this.emit('process', { | |||
processType: 'summarize-transcript', | |||
phase: 'success', | |||
data: summary, | |||
content: summary, | |||
contentType: 'text/plain', | |||
}); | |||