mirror of
https://github.com/magnus919/agent-skills.git
synced 2026-09-11 19:47:12 +03:00
Magnus Hedemark
fe5b275d00
Greenfield SkillOpt: 3 epochs for deepset Haystack skill. Pipeline DAG model, document stores, retrievers, evaluation, deployment. Epoch 1 — Prominence: Hard-gate on Pipeline DAG vs LCEL pipe model Epoch 2 — Decision Guidance: Where to Start, Framework Routing Guide Epoch 3 — Pattern Expansion: Hybrid RAG pattern, evaluation pipeline, deployment 11 files: SKILL.md, 6 references, 3 templates, 1 script.
25 lines
1.1 KiB
Python
25 lines
1.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Haystack indexing pipeline — load, split, embed, write."""
|
|
|
|
from haystack import Pipeline
|
|
from haystack.components.converters import TextFileToDocument
|
|
from haystack.components.preprocessors import DocumentSplitter
|
|
from haystack.components.embedders import SentenceTransformersDocumentEmbedder
|
|
from haystack.components.writers import DocumentWriter
|
|
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
|
|
|
document_store = InMemoryDocumentStore()
|
|
|
|
pipeline = Pipeline()
|
|
pipeline.add_component("converter", TextFileToDocument())
|
|
pipeline.add_component("splitter", DocumentSplitter(split_by="word", split_length=500, split_overlap=50))
|
|
pipeline.add_component("embedder", SentenceTransformersDocumentEmbedder())
|
|
pipeline.add_component("writer", DocumentWriter(document_store=document_store))
|
|
|
|
pipeline.connect("converter.documents", "splitter.documents")
|
|
pipeline.connect("splitter.documents", "embedder.documents")
|
|
pipeline.connect("embedder.documents", "writer.documents")
|
|
|
|
result = pipeline.run({"converter": {"sources": ["docs.txt"]}})
|
|
print(f"Indexed {document_store.count_documents()} documents")
|