Procházet zdrojové kódy

Use env variable for prompts dir, refactor events

The prompts directory is now specified as an environment variable to be
more compatible with custom locations.

The events have also been refactored.
master
TheoryOfNekomata před 1 rokem
rodič
revize
796a8ff49b
6 změnil soubory, kde provedl 89 přidání a 41 odebrání
  1. +3
    -0
      .env.example
  2. +8
    -2
      src/common.ts
  3. +5
    -0
      src/config.ts
  4. +2
    -4
      src/index.ts
  5. +60
    -22
      src/summarizer.ts
  6. +11
    -13
      src/video-types/youtube/index.ts

+ 3
- 0
.env.example Zobrazit soubor

@@ -3,3 +3,6 @@ OPENAI_API_KEY=

# OpenAI organization ID.
OPENAI_ORGANIZATION_ID=

# Directory where the prompts are stored.
OPENAI_PROMPTS_DIR=

+ 8
- 2
src/common.ts Zobrazit soubor

@@ -17,10 +17,16 @@ export interface SummarizerEventEmitter extends NodeJS.EventEmitter {
on(eventType: 'end', callback: () => void): this;
}

export interface OpenAiParams {
apiKey: string;
organizationId?: string;
model?: string;
temperature?: number;
}

export interface CreateBaseSummarizerParams {
url: string;
language?: string;
country?: string;
openaiApiKey: string;
openaiOrganizationId?: string;
openAiParams: OpenAiParams;
}

+ 5
- 0
src/config.ts Zobrazit soubor

@@ -0,0 +1,5 @@
export namespace openAi {
export const apiKey = process.env.OPENAI_API_KEY as string;
export const organizationId = process.env.OPENAI_ORGANIZATION_ID;
export const promptsDir = process.env.OPENAI_PROMPTS_DIR as string;
}

+ 2
- 4
src/index.ts Zobrazit soubor

@@ -16,8 +16,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven
const {
type: videoType,
url,
openaiOrganizationId,
openaiApiKey,
openAiParams,
language,
country,
} = params;
@@ -26,8 +25,7 @@ export const createSummarizer = (params: CreateSummarizerParams): SummarizerEven
case VideoType.YOUTUBE:
return new YouTubeSummarizerEventEmitter({
url,
openaiOrganizationId,
openaiApiKey,
openAiParams,
language,
country,
});


+ 60
- 22
src/summarizer.ts Zobrazit soubor

@@ -2,8 +2,32 @@ import fetchPonyfill from 'fetch-ponyfill';
import Handlebars from 'handlebars';
import { resolve } from 'path';
import { readFile } from 'fs/promises';
import * as config from './config';
import { OpenAiParams } from './common';

export interface MakeAiCallParams {
prompts: string[];
openAiParams: OpenAiParams;
}

export class AiCallError extends Error {
constructor(message: string, public readonly response: Response) {
super(message);
this.name = 'AiCallError';
}
}

const makeAiCall = async (params: MakeAiCallParams): Promise<string> => {
const {
prompts,
openAiParams: {
apiKey,
organizationId,
model = 'gpt-3.5-turbo',
temperature = 0.6,
},
} = params;

const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: string): Promise<string> => {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
Accept: 'application/json',
@@ -21,9 +45,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
method: 'POST',
headers,
body: JSON.stringify({
//model: 'gpt-4',
model: 'gpt-3.5-turbo',
temperature: 0.6,
model,
temperature,
messages: [
{
role: 'user',
@@ -35,9 +58,8 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
);

if (!response.ok) {
const responseText = await response.text();
console.log(responseText);
throw new Error(`OpenAI API call failed with status ${response.status}`);
const { error } = await response.json();
throw new AiCallError(`OpenAI API call failed with status ${response.status}: ${error.message}`, response);
}

const { choices } = await response.json();
@@ -47,38 +69,54 @@ const makeAiCall = async (prompts: string[], apiKey: string, organizationId?: st
};

const compilePrompts = async (filename: string, params: Record<string, unknown>): Promise<string[]> => {
const rawPromptText = await readFile(resolve(__dirname, filename), 'utf-8');
const rawPromptText = await readFile(resolve(config.openAi.promptsDir, filename), 'utf-8');
const fill = Handlebars.compile(rawPromptText, { noEscape: true });
const filledText = fill(params);
return filledText.split('---').map((s) => s.trim());
};

export const normalizeTranscriptText = async (
export interface NormalizeTranscriptTextParams {
rawTranscriptText: string,
apiKey: string,
organizationId?: string,
) => {
openAiParams: OpenAiParams,
}

export const normalizeTranscriptText = async (params: NormalizeTranscriptTextParams) => {
const {
rawTranscriptText,
openAiParams,
} = params;
const prompts = await compilePrompts(
'../prompts/normalize-transcript-text.hbs',
'normalize-transcript-text.hbs',
{
transcript: rawTranscriptText,
},
);

return makeAiCall(prompts, apiKey, organizationId);
return makeAiCall({
prompts,
openAiParams,
});
};

export const summarizeTranscript = async (
transcript: string,
apiKey: string,
organizationId?: string,
) => {
export interface SummarizeTranscriptParams {
normalizedTranscript: string,
openAiParams: OpenAiParams,
}

export const summarizeTranscript = async (params: SummarizeTranscriptParams) => {
const {
normalizedTranscript,
openAiParams,
} = params;
const prompts = await compilePrompts(
'../prompts/summarize-transcript.hbs',
'summarize-transcript.hbs',
{
transcript,
transcript: normalizedTranscript,
},
);

return makeAiCall(prompts, apiKey, organizationId);
return makeAiCall({
prompts,
openAiParams,
});
};

+ 11
- 13
src/video-types/youtube/index.ts Zobrazit soubor

@@ -8,7 +8,7 @@ import {
} from './transcript';
import { normalizeTranscriptText, summarizeTranscript } from '../../summarizer';

export interface CreateYouTubeSummarizerParams extends CreateBaseSummarizerParams {}
export type CreateYouTubeSummarizerParams = CreateBaseSummarizerParams

export class YouTubeSummarizerEventEmitter extends EventEmitter implements SummarizerEventEmitter {
constructor(private readonly params: CreateYouTubeSummarizerParams) {
@@ -18,8 +18,7 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
process() {
const {
url,
openaiApiKey,
openaiOrganizationId,
openAiParams,
...config
} = this.params;
const identifier = retrieveVideoId(url);
@@ -52,21 +51,20 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
});

this.emit('process', {
processType: 'normalize-caption',
processType: 'normalize-transcript',
phase: 'start',
});

return normalizeTranscriptText(
transcript.map((item) => item.text).join(' '),
openaiApiKey,
openaiOrganizationId,
);
return normalizeTranscriptText({
rawTranscriptText: transcript.map((item) => item.text).join(' '),
openAiParams,
});
})
.then((normalizedCaption) => {
.then((normalizedTranscript) => {
this.emit('process', {
processType: 'normalize-transcript',
phase: 'success',
content: normalizedCaption,
content: normalizedTranscript,
contentType: 'text/plain',
});

@@ -75,13 +73,13 @@ export class YouTubeSummarizerEventEmitter extends EventEmitter implements Summa
phase: 'start',
});

return summarizeTranscript(normalizedCaption, openaiApiKey, openaiOrganizationId);
return summarizeTranscript({ normalizedTranscript, openAiParams });
})
.then((summary) => {
this.emit('process', {
processType: 'summarize-transcript',
phase: 'success',
data: summary,
content: summary,
contentType: 'text/plain',
});



Načítá se…
Zrušit
Uložit