diff --git a/.changeset/full-views-wonder.md b/.changeset/full-views-wonder.md new file mode 100644 index 00000000000..fd9f9d0b93a --- /dev/null +++ b/.changeset/full-views-wonder.md @@ -0,0 +1,5 @@ +--- +'@mastra/rag': patch +--- + +Updated KeywordExtraction in chunk diff --git a/docs/src/pages/docs/reference/rag/chunk.mdx b/docs/src/pages/docs/reference/rag/chunk.mdx index 3289c5f631b..9242f962b09 100644 --- a/docs/src/pages/docs/reference/rag/chunk.mdx +++ b/docs/src/pages/docs/reference/rag/chunk.mdx @@ -31,10 +31,8 @@ const chunksWithMetadata = await doc.chunk({ strategy: 'markdown', headers: [['#', 'title'], ['##', 'section']], extract: { - fields: [ - { name: 'summary', description: 'A brief summary of the chunk content' }, - { name: 'keywords', description: 'Key terms found in the chunk' } - ] + summary: true, // Extract summaries with default settings + keywords: true // Extract keywords with default settings } }); ``` diff --git a/docs/src/pages/docs/reference/rag/document.mdx b/docs/src/pages/docs/reference/rag/document.mdx index 3d89b2673e6..aca200d7413 100644 --- a/docs/src/pages/docs/reference/rag/document.mdx +++ b/docs/src/pages/docs/reference/rag/document.mdx @@ -115,10 +115,8 @@ const chunks = await doc.chunk({ strategy: 'markdown', headers: [['#', 'title'], ['##', 'section']], extract: { - fields: [ - { name: 'summary', description: 'A brief summary' }, - { name: 'keywords', description: 'Key terms' } - ] + summary: true, // Extract summaries with default settings + keywords: true // Extract keywords with default settings } }); diff --git a/docs/src/pages/docs/reference/rag/extract-params.mdx b/docs/src/pages/docs/reference/rag/extract-params.mdx index fde1c954d22..cea36aa36ef 100644 --- a/docs/src/pages/docs/reference/rag/extract-params.mdx +++ b/docs/src/pages/docs/reference/rag/extract-params.mdx @@ -5,70 +5,222 @@ description: Documentation for metadata extraction configuration in Mastra. # ExtractParams -ExtractParams configures metadata extraction from document chunks. +ExtractParams configures metadata extraction from document chunks using LLM analysis. ## Example -## ExtractParams - -`ExtractParams` configures automatic metadata extraction from chunks using LLM analysis. - ```typescript showLineNumbers copy import { MDocument } from "@mastra/rag"; const doc = MDocument.fromText(text); const chunks = await doc.chunk({ extract: { - fields: [ - { - name: 'summary', - description: 'A 1-2 sentence summary of the main points' - }, - { - name: 'entities', - description: 'List of companies, people, and locations mentioned' - }, - { - name: 'custom_field', - description: 'Any other metadata you want to extract, guided by this description' - } - ], - model: 'gpt-4o-mini' // Optional: specify a different model + title: true, // Extract titles using default settings + summary: true, // Generate summaries using default settings + keywords: true // Extract keywords using default settings } }); + +// Example output: +// chunks[0].metadata = { +// documentTitle: "AI Systems Overview", +// sectionSummary: "Overview of artificial intelligence concepts and applications", +// excerptKeywords: "KEYWORDS: AI, machine learning, algorithms" +// } ``` ## Parameters +The `extract` parameter accepts the following fields: + ", - description: "Array of fields to extract from each chunk", - isOptional: false + name: "title", + type: "boolean | TitleExtractorsArgs", + isOptional: true, + description: + "Enable title extraction. Set to true for default settings, or provide custom configuration.", }, { - name: "model", + name: "summary", + type: "boolean | SummaryExtractArgs", + isOptional: true, + description: + "Enable summary extraction. Set to true for default settings, or provide custom configuration.", + }, + { + name: "questions", + type: "boolean | QuestionAnswerExtractArgs", + isOptional: true, + description: + "Enable question generation. Set to true for default settings, or provide custom configuration.", + }, + { + name: "keywords", + type: "boolean | KeywordExtractArgs", + isOptional: true, + description: + "Enable keyword extraction. Set to true for default settings, or provide custom configuration.", + }, + ]} +/> + +## Extractor Arguments + +### TitleExtractorsArgs + + +/> -## Field Types +### SummaryExtractArgs -The fields are flexible - you can define any metadata fields you want to extract. Common field types include: + -- `summary`: Brief overview of chunk content -- `keywords`: Key terms or concepts -- `topics`: Main subjects discussed -- `entities`: Named entities (people, places, organizations) -- `sentiment`: Emotional tone -- `language`: Detected language -- `timestamp`: Temporal references -- `categories`: Content classification +### QuestionAnswerExtractArgs -Example: + + +### KeywordExtractArgs + + + +## Advanced Example + +```typescript showLineNumbers copy +import { MDocument } from "@mastra/rag"; + +const doc = MDocument.fromText(text); +const chunks = await doc.chunk({ + extract: { + // Title extraction with custom settings + title: { + nodes: 2, // Extract 2 title nodes + nodeTemplate: "Generate a title for this: {context}", + combineTemplate: "Combine these titles: {context}" + }, + + // Summary extraction with custom settings + summary: { + summaries: ["self"], // Generate summaries for current chunk + promptTemplate: "Summarize this: {context}" + }, + + // Question generation with custom settings + questions: { + questions: 3, // Generate 3 questions + promptTemplate: "Generate {numQuestions} questions about: {context}", + embeddingOnly: false + }, + + // Keyword extraction with custom settings + keywords: { + keywords: 5, // Extract 5 keywords + promptTemplate: "Extract {maxKeywords} key terms from: {context}" + } + } +}); + +// Example output: +// chunks[0].metadata = { +// documentTitle: "AI in Modern Computing", +// sectionSummary: "Overview of AI concepts and their applications in computing", +// questionsThisExcerptCanAnswer: "1. What is machine learning?\n2. How do neural networks work?", +// excerptKeywords: "1. Machine learning\n2. Neural networks\n3. Training data" +// } +``` diff --git a/packages/rag/src/document/document.test.ts b/packages/rag/src/document/document.test.ts index 69e68a6c856..776d6ab91a2 100644 --- a/packages/rag/src/document/document.test.ts +++ b/packages/rag/src/document/document.test.ts @@ -1638,6 +1638,79 @@ describe('MDocument', () => { expect(docs?.[0]?.text).toContain('# Title'); }); }); + + describe('metadata extraction', () => { + it('should extract metadata with default settings', async () => { + const doc = MDocument.fromMarkdown( + '# AI and Machine Learning\n\nThis is a test document about artificial intelligence and machine learning.', + ); + + const chunks = await doc.chunk({ + strategy: 'markdown', + extract: { + title: true, + summary: true, + keywords: true, + }, + }); + + const metadata = chunks[0].metadata; + expect(metadata).toBeDefined(); + expect(metadata.documentTitle).toBeDefined(); + expect(metadata.sectionSummary).toBeDefined(); + expect(metadata.excerptKeywords).toMatch(/^KEYWORDS: .*/); + }, 15000); + + it('should extract metadata with custom settings', async () => { + const doc = MDocument.fromMarkdown( + '# AI and Machine Learning\n\nThis is a test document about artificial intelligence and machine learning.', + ); + + const chunks = await doc.chunk({ + strategy: 'markdown', + extract: { + title: { + nodes: 2, + nodeTemplate: 'Generate a title for this: {context}', + combineTemplate: 'Combine these titles: {context}', + }, + summary: { + summaries: ['self'], + promptTemplate: 'Summarize this: {context}', + }, + questions: { + questions: 2, + promptTemplate: 'Generate {numQuestions} questions about: {context}', + }, + keywords: { + keywords: 3, + promptTemplate: 'Extract {maxKeywords} key terms from: {context}', + }, + }, + }); + + const metadata = chunks[0].metadata; + expect(metadata).toBeDefined(); + expect(metadata.documentTitle).toBeDefined(); + expect(metadata.sectionSummary).toBeDefined(); + expect(metadata.questionsThisExcerptCanAnswer).toMatch(/^1\. .*\?2\. .*\?$/); + expect(metadata.excerptKeywords).toMatch(/^1\. .*\n2\. .*\n3\. .*$/); + }, 15000); + + it('should handle invalid summary types', async () => { + const doc = MDocument.fromText('Test document'); + + await expect( + doc.chunk({ + extract: { + summary: { + summaries: ['invalid'], + }, + }, + }), + ).rejects.toThrow("Summaries must be one of 'self', 'prev', 'next'"); + }, 15000); + }); }); // Helper function to find the longest common substring between two strings diff --git a/packages/rag/src/document/document.ts b/packages/rag/src/document/document.ts index ef4b83b8e0f..81190c808af 100644 --- a/packages/rag/src/document/document.ts +++ b/packages/rag/src/document/document.ts @@ -5,6 +5,8 @@ import { QuestionsAnsweredExtractor, SummaryExtractor, TitleExtractor, + ObjectType, + NodeRelationship, } from 'llamaindex'; import { CharacterTransformer, RecursiveCharacterTransformer } from './transformers/character'; @@ -43,6 +45,19 @@ export class MDocument { if (typeof title !== 'undefined') { transformations.push(new TitleExtractor(typeof title === 'boolean' ? {} : title)); + this.chunks = this.chunks.map( + (doc, i) => + new Chunk({ + ...doc, + relationships: { + [NodeRelationship.SOURCE]: { + nodeId: `doc-${i}`, + nodeType: ObjectType.DOCUMENT, + metadata: doc.metadata, + }, + }, + }), + ); } const pipeline = new IngestionPipeline({ diff --git a/packages/rag/src/document/types.ts b/packages/rag/src/document/types.ts index a5d6b00e0bb..0e5623458ba 100644 --- a/packages/rag/src/document/types.ts +++ b/packages/rag/src/document/types.ts @@ -41,7 +41,7 @@ export type ExtractParams = { title?: TitleExtractorsArgs | boolean; summary?: SummaryExtractArgs | boolean; questions?: QuestionAnswerExtractArgs | boolean; - keywords?: boolean | Record; + keywords?: KeywordExtractArgs | boolean; }; export type ChunkOptions = {