Przeglądaj źródła

Use env variable for prompts dir, refactor events

The prompts directory is now specified as an environment variable to be
more compatible with custom locations.

The events have also been refactored.
master
TheoryOfNekomata 1 rok temu
rodzic
commit
796a8ff49b
6 zmienionych plików z 89 dodań i 41 usunięć
  1. +3
    -0
      .env.example
  2. +8
    -2
      src/common.ts
  3. +5
    -0
      src/config.ts
  4. +2
    -4
      src/index.ts
  5. +60
    -22
      src/summarizer.ts
  6. +11
    -13
      src/video-types/youtube/index.ts

+ 3
- 0
.env.example Wyświetl plik

@@ -3,3 +3,6 @@ OPENAI_API_KEY=

# OpenAI organization ID.
OPENAI_ORGANIZATION_ID=

# Directory where the prompts are stored.
OPENAI_PROMPTS_DIR=

+ 8
- 2
src/common.ts Wyświetl plik

@@ -17,10 +17,16 @@ export interface SummarizerEventEmitter extends NodeJS.EventEmitter {
on(eventType: 'end', callback: () => void): this;
}

export interface OpenAiParams {
apiKey: string;
organizationId?: string;
model?: string;
temperature?: number;
}

export interface CreateBaseSummarizerParams {
url: string;
language?: string;
country?: string;
openaiApiKey: string;
openaiOrganizationId?: string;
openAiParams: OpenAiParams;
}

+ 5
- 0
src/config.ts Wyświetl plik

@@ -0,0 +1,5 @@
export namespace openAi {
export const apiKey = process.env.OPENAI_API_KEY as string;
export const organizationId = process.env.OPENAI_ORGANIZATION_ID;
export const promptsDir = process.env.OPENAI_PROMPTS_DIR as string;
}

+ 2
- 4
src/index.ts Wyświetl plik

@@ -16,8 +16,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven
const {
type: videoType,
url,
openaiOrganizationId,
openaiApiKey,
openAiParams,
language,
country,
} = params;
@@ -26,8 +25,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven
case VideoType.YOUTUBE:
return new YouTubeSummarizerEventEmitter({
url,
openaiOrganizationId,
openaiApiKey,
openAiParams,
language,
country,
});


+ 60
- 22
src/summarizer.ts Wyświetl plik

@@ -2,8 +2,32 @@ import fetchPonyfill from 'fetch-ponyfill';
import Handlebars from 'handlebars';
import { resolve } from 'path';
import { readFile } from 'fs/promises';
import * as config from './config';
import { OpenAiParams } from './common';

export interface MakeAiCallParams {
prompts: string[];
openAiParams: OpenAiParams;
}

export class AiCallError extends Error {
constructor(message: string, public readonly response: Response) {
super(message);
this.name = 'AiCallError';
}
}

const makeAiCall = async (params: MakeAiCallParams): Promise<string> => {
const {
prompts,
openAiParams: {
apiKey,
organizationId,
model = 'gpt-3.5-turbo',
temperature = 0.6,
},
} = params;

const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: string): Promise<string> => {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
Accept: 'application/json',
@@ -21,9 +45,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
method: 'POST',
headers,
body: JSON.stringify({
//model: 'gpt-4',
model: 'gpt-3.5-turbo',
temperature: 0.6,
model,
temperature,
messages: [
{
role: 'user',
@@ -35,9 +58,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
);

if (!response.ok) {
const responseText = await response.text();
console.log(responseText);
throw new Error(`OpenAI API call failed with status ${response.status}`);
const { error } = await response.json();
throw new AiCallError(`OpenAI API call failed with status ${response.status}: ${error.message}`, response);
}

const { choices } = await response.json();
@@ -47,38 +69,54 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
};

const compilePrompts = async (filename: string, params: Record<string, unknown>): Promise<string[]> => {
const rawPromptText = await readFile(resolve(__dirname, filename), 'utf-8');
const rawPromptText = await readFile(resolve(config.openAi.promptsDir, filename), 'utf-8');
const fill = Handlebars.compile(rawPromptText, { noEscape: true });
const filledText = fill(params);
return filledText.split('---').map((s) => s.trim());
};

export const normalizeTranscriptText = async (
export interface NormalizeTranscriptTextParams {
rawTranscriptText: string,
apiKey: string,
organizationId?: string,
) => {
openAiParams: OpenAiParams,
}

export const normalizeTranscriptText = async (params: NormalizeTranscriptTextParams) => {
const {
rawTranscriptText,
openAiParams,
} = params;
const prompts = await compilePrompts(
'../prompts/normalize-transcript-text.hbs',
'normalize-transcript-text.hbs',
{
transcript: rawTranscriptText,
},
);

return makeAiCall(prompts, apiKey, organizationId);
return makeAiCall({
prompts,
openAiParams,
});
};

export const summarizeTranscript = async (
transcript: string,
apiKey: string,
organizationId?: string,
) => {
export interface SummarizeTranscriptParams {
normalizedTranscript: string,
openAiParams: OpenAiParams,
}

export const summarizeTranscript = async (params: SummarizeTranscriptParams) => {
const {
normalizedTranscript,
openAiParams,
} = params;
const prompts = await compilePrompts(
'../prompts/summarize-transcript.hbs',
'summarize-transcript.hbs',
{
transcript,
transcript: normalizedTranscript,
},
);

return makeAiCall(prompts, apiKey, organizationId);
return makeAiCall({
prompts,
openAiParams,
});
};

+ 11
- 13
src/video-types/youtube/index.ts Wyświetl plik

@@ -8,7 +8,7 @@ import {
} from './transcript';
import { normalizeTranscriptText, summarizeTranscript } from '../../summarizer';

export interface CreateYouTubeSummarizerParams extends CreateBaseSummarizerParams {}
export type CreateYouTubeSummarizerParams = CreateBaseSummarizerParams

export class YouTubeSummarizerEventEmitter extends EventEmitter implements SummarizerEventEmitter {
constructor(private readonly params: CreateYouTubeSummarizerParams) {
@@ -18,8 +18,7 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
process() {
const {
url,
openaiApiKey,
openaiOrganizationId,
openAiParams,
...config
} = this.params;
const identifier = retrieveVideoId(url);
@@ -52,21 +51,20 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
});

this.emit('process', {
processType: 'normalize-caption',
processType: 'normalize-transcript',
phase: 'start',
});

return normalizeTranscriptText(
transcript.map((item) => item.text).join(' '),
openaiApiKey,
openaiOrganizationId,
);
return normalizeTranscriptText({
rawTranscriptText: transcript.map((item) => item.text).join(' '),
openAiParams,
});
})
.then((normalizedCaption) => {
.then((normalizedTranscript) => {
this.emit('process', {
processType: 'normalize-transcript',
phase: 'success',
content: normalizedCaption,
content: normalizedTranscript,
contentType: 'text/plain',
});

@@ -75,13 +73,13 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
phase: 'start',
});

return summarizeTranscript(normalizedCaption, openaiApiKey, openaiOrganizationId);
return summarizeTranscript({ normalizedTranscript, openAiParams });
})
.then((summary) => {
this.emit('process', {
processType: 'summarize-transcript',
phase: 'success',
data: summary,
content: summary,
contentType: 'text/plain',
});



Ładowanie…
Anuluj
Zapisz