Browse Source

Use env variable for prompts dir, refactor events

The prompts directory is now specified as an environment variable to be
more compatible with custom locations.

The events have also been refactored.
master
TheoryOfNekomata 1 year ago
parent
commit
796a8ff49b
6 changed files with 89 additions and 41 deletions
  1. +3
    -0
      .env.example
  2. +8
    -2
      src/common.ts
  3. +5
    -0
      src/config.ts
  4. +2
    -4
      src/index.ts
  5. +60
    -22
      src/summarizer.ts
  6. +11
    -13
      src/video-types/youtube/index.ts

+ 3
- 0
.env.example View File

@@ -3,3 +3,6 @@ OPENAI_API_KEY=

# OpenAI organization ID.
OPENAI_ORGANIZATION_ID=

# Directory where the prompts are stored.
OPENAI_PROMPTS_DIR=

+ 8
- 2
src/common.ts View File

@@ -17,10 +17,16 @@ export interface SummarizerEventEmitter extends NodeJS.EventEmitter {
on(eventType: 'end', callback: () => void): this;
}

export interface OpenAiParams {
apiKey: string;
organizationId?: string;
model?: string;
temperature?: number;
}

export interface CreateBaseSummarizerParams {
url: string;
language?: string;
country?: string;
openaiApiKey: string;
openaiOrganizationId?: string;
openAiParams: OpenAiParams;
}

+ 5
- 0
src/config.ts View File

@@ -0,0 +1,5 @@
export namespace openAi {
export const apiKey = process.env.OPENAI_API_KEY as string;
export const organizationId = process.env.OPENAI_ORGANIZATION_ID;
export const promptsDir = process.env.OPENAI_PROMPTS_DIR as string;
}

+ 2
- 4
src/index.ts View File

@@ -16,8 +16,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven
const {
type: videoType,
url,
openaiOrganizationId,
openaiApiKey,
openAiParams,
language,
country,
} = params;
@@ -26,8 +25,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven
case VideoType.YOUTUBE:
return new YouTubeSummarizerEventEmitter({
url,
openaiOrganizationId,
openaiApiKey,
openAiParams,
language,
country,
});


+ 60
- 22
src/summarizer.ts View File

@@ -2,8 +2,32 @@ import fetchPonyfill from 'fetch-ponyfill';
import Handlebars from 'handlebars';
import { resolve } from 'path';
import { readFile } from 'fs/promises';
import * as config from './config';
import { OpenAiParams } from './common';

export interface MakeAiCallParams {
prompts: string[];
openAiParams: OpenAiParams;
}

export class AiCallError extends Error {
constructor(message: string, public readonly response: Response) {
super(message);
this.name = 'AiCallError';
}
}

const makeAiCall = async (params: MakeAiCallParams): Promise<string> => {
const {
prompts,
openAiParams: {
apiKey,
organizationId,
model = 'gpt-3.5-turbo',
temperature = 0.6,
},
} = params;

const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: string): Promise<string> => {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
Accept: 'application/json',
@@ -21,9 +45,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
method: 'POST',
headers,
body: JSON.stringify({
//model: 'gpt-4',
model: 'gpt-3.5-turbo',
temperature: 0.6,
model,
temperature,
messages: [
{
role: 'user',
@@ -35,9 +58,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
);

if (!response.ok) {
const responseText = await response.text();
console.log(responseText);
throw new Error(`OpenAI API call failed with status ${response.status}`);
const { error } = await response.json();
throw new AiCallError(`OpenAI API call failed with status ${response.status}: ${error.message}`, response);
}

const { choices } = await response.json();
@@ -47,38 +69,54 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
};

const compilePrompts = async (filename: string, params: Record<string, unknown>): Promise<string[]> => {
const rawPromptText = await readFile(resolve(__dirname, filename), 'utf-8');
const rawPromptText = await readFile(resolve(config.openAi.promptsDir, filename), 'utf-8');
const fill = Handlebars.compile(rawPromptText, { noEscape: true });
const filledText = fill(params);
return filledText.split('---').map((s) => s.trim());
};

export const normalizeTranscriptText = async (
export interface NormalizeTranscriptTextParams {
rawTranscriptText: string,
apiKey: string,
organizationId?: string,
) => {
openAiParams: OpenAiParams,
}

export const normalizeTranscriptText = async (params: NormalizeTranscriptTextParams) => {
const {
rawTranscriptText,
openAiParams,
} = params;
const prompts = await compilePrompts(
'../prompts/normalize-transcript-text.hbs',
'normalize-transcript-text.hbs',
{
transcript: rawTranscriptText,
},
);

return makeAiCall(prompts, apiKey, organizationId);
return makeAiCall({
prompts,
openAiParams,
});
};

export const summarizeTranscript = async (
transcript: string,
apiKey: string,
organizationId?: string,
) => {
export interface SummarizeTranscriptParams {
normalizedTranscript: string,
openAiParams: OpenAiParams,
}

export const summarizeTranscript = async (params: SummarizeTranscriptParams) => {
const {
normalizedTranscript,
openAiParams,
} = params;
const prompts = await compilePrompts(
'../prompts/summarize-transcript.hbs',
'summarize-transcript.hbs',
{
transcript,
transcript: normalizedTranscript,
},
);

return makeAiCall(prompts, apiKey, organizationId);
return makeAiCall({
prompts,
openAiParams,
});
};

+ 11
- 13
src/video-types/youtube/index.ts View File

@@ -8,7 +8,7 @@ import {
} from './transcript';
import { normalizeTranscriptText, summarizeTranscript } from '../../summarizer';

export interface CreateYouTubeSummarizerParams extends CreateBaseSummarizerParams {}
export type CreateYouTubeSummarizerParams = CreateBaseSummarizerParams

export class YouTubeSummarizerEventEmitter extends EventEmitter implements SummarizerEventEmitter {
constructor(private readonly params: CreateYouTubeSummarizerParams) {
@@ -18,8 +18,7 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
process() {
const {
url,
openaiApiKey,
openaiOrganizationId,
openAiParams,
...config
} = this.params;
const identifier = retrieveVideoId(url);
@@ -52,21 +51,20 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
});

this.emit('process', {
processType: 'normalize-caption',
processType: 'normalize-transcript',
phase: 'start',
});

return normalizeTranscriptText(
transcript.map((item) => item.text).join(' '),
openaiApiKey,
openaiOrganizationId,
);
return normalizeTranscriptText({
rawTranscriptText: transcript.map((item) => item.text).join(' '),
openAiParams,
});
})
.then((normalizedCaption) => {
.then((normalizedTranscript) => {
this.emit('process', {
processType: 'normalize-transcript',
phase: 'success',
content: normalizedCaption,
content: normalizedTranscript,
contentType: 'text/plain',
});

@@ -75,13 +73,13 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
phase: 'start',
});

return summarizeTranscript(normalizedCaption, openaiApiKey, openaiOrganizationId);
return summarizeTranscript({ normalizedTranscript, openAiParams });
})
.then((summary) => {
this.emit('process', {
processType: 'summarize-transcript',
phase: 'success',
data: summary,
content: summary,
contentType: 'text/plain',
});



Loading…
Cancel
Save