{"componentChunkName":"component---src-gatsby-theme-chronoblog-templates-note-js","path":"/notes/langchain-loaders-chunking-nodejs/","result":{"data":{"mdx":{"parent":{"__typename":"File","fields":{"gitLogLatestDate":"2026-06-16 00:19:36 +0200"}},"id":"d12dd98f-1f01-5a2a-ad8b-faa09a10956b","excerpt":"This post covers local file ingestion and chunking in Node.js. For LangChain basics (LCEL, packages, agents), see the  LangChain overview…","frontmatter":{"title":"Document loaders and chunking with LangChain","date":"2026-06-16 00:01:00 UTC","job_ad":null,"job_ad_id":null,"job_ad_url":null,"tags":["langchain","rag","ai","node"],"cover":{"childImageSharp":{"fluid":{"base64":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAApUlEQVQoz71RiwrCMAzs/3/mZN2cuqb2/TyxIDpx4GAaOHJJyIVwDDsHq7U2cs+f8JhtEnziXXj90BrY7i8LEWF0hvcFYi7oe2qYLxnOVViTQSJC64wYCiQlEEV4V+BsBlGCpNiy9xWMcwUSAUZHnE8Zh47QdYRpSk1EUgDvFcQcYG3CdLQYuIZWCVcZMA4G46Baz9ryg5eXji75a/21KVsX/u7yDZPtFlE9ZuxUAAAAAElFTkSuQmCC","aspectRatio":2.0869565217391304,"src":"/static/749edec72506a418aab9de612ee03217/c4ecb/cover.png","srcSet":"/static/749edec72506a418aab9de612ee03217/57ab0/cover.png 192w,\n/static/749edec72506a418aab9de612ee03217/f4739/cover.png 384w,\n/static/749edec72506a418aab9de612ee03217/c4ecb/cover.png 768w","srcWebp":"/static/749edec72506a418aab9de612ee03217/dd090/cover.webp","srcSetWebp":"/static/749edec72506a418aab9de612ee03217/ae504/cover.webp 192w,\n/static/749edec72506a418aab9de612ee03217/fef30/cover.webp 384w,\n/static/749edec72506a418aab9de612ee03217/dd090/cover.webp 768w","sizes":"(max-width: 768px) 100vw, 768px","presentationWidth":768,"presentationHeight":366},"resize":{"src":"/static/749edec72506a418aab9de612ee03217/c4ecb/cover.png"}}}},"fields":{"slug":"/notes/langchain-loaders-chunking-nodejs/","readingTime":{"text":"4 min read"}},"body":"function _extends() { _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; return _extends.apply(this, arguments); }\n\nfunction _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }\n\nfunction _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }\n\n/* @jsxRuntime classic */\n\n/* @jsx mdx */\nvar _frontmatter = {\n  \"title\": \"Document loaders and chunking with LangChain\",\n  \"date\": \"2026-06-16 00:01:00 UTC\",\n  \"cover\": \"./cover.png\",\n  \"tags\": [\"langchain\", \"rag\", \"ai\", \"node\"],\n  \"canonical_url\": \"https://sevic.dev/notes/langchain-loaders-chunking-nodejs/\"\n};\nvar layoutProps = {\n  _frontmatter: _frontmatter\n};\nvar MDXLayout = \"wrapper\";\nreturn function MDXContent(_ref) {\n  var components = _ref.components,\n      props = _objectWithoutProperties(_ref, [\"components\"]);\n\n  return mdx(MDXLayout, _extends({}, layoutProps, props, {\n    components: components,\n    mdxType: \"MDXLayout\"\n  }), mdx(\"p\", null, \"This post covers local file ingestion and chunking in Node.js. For LangChain basics (LCEL, packages, agents), see the \", mdx(\"a\", _extends({\n    parentName: \"p\"\n  }, {\n    \"href\": \"https://sevic.dev/notes/langchain-overview-nodejs/\"\n  }), \"LangChain overview post\"), \". For embeddings, pgvector, and the full RAG flow, see the \", mdx(\"a\", _extends({\n    parentName: \"p\"\n  }, {\n    \"href\": \"https://sevic.dev/notes/rag-openai-embeddings-pgvector-langchain/\"\n  }), \"RAG with pgvector post\"), \" - it uses one splitter inline; this post goes deeper on loaders and splitter choice.\"), mdx(\"h3\", {\n    \"id\": \"prerequisites\"\n  }, \"Prerequisites\"), mdx(\"ul\", null, mdx(\"li\", {\n    parentName: \"ul\"\n  }, \"Node.js version 26\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"langchain\"), \", \", mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"@langchain/core\"), \", \", mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"@langchain/classic\"), \", and \", mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"@langchain/textsplitters\"), \" installed\")), mdx(\"pre\", null, mdx(\"code\", _extends({\n    parentName: \"pre\"\n  }, {\n    \"className\": \"language-bash\"\n  }), \"npm i langchain @langchain/core @langchain/classic @langchain/textsplitters\\n\")), mdx(\"p\", null, \"More loader types (web, cloud, audio) live in standalone integration packages - see the \", mdx(\"a\", _extends({\n    parentName: \"p\"\n  }, {\n    \"href\": \"https://docs.langchain.com/oss/javascript/integrations/document_loaders\"\n  }), \"document loader integrations\"), \" page.\"), mdx(\"h3\", {\n    \"id\": \"the-document-type\"\n  }, \"The Document type\"), mdx(\"p\", null, \"Every loader returns \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"Document\"), \" instances from \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"@langchain/core\"), \":\"), mdx(\"ul\", null, mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"pageContent\"), \" - the text of the chunk or file\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"metadata\"), \" - optional key/value pairs (source path, section, page) used for citations\")), mdx(\"pre\", null, mdx(\"code\", _extends({\n    parentName: \"pre\"\n  }, {\n    \"className\": \"language-js\"\n  }), \"import { Document } from '@langchain/core/documents';\\n\\nconst doc = new Document({\\n  pageContent: 'pgvector adds vector search to PostgreSQL.',\\n  metadata: { source: 'notes/pgvector.txt', section: 'basics' }\\n});\\n\")), mdx(\"h3\", {\n    \"id\": \"load-a-single-file\"\n  }, \"Load a single file\"), mdx(\"p\", null, \"Use \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"TextLoader\"), \" for plain text or markdown files:\"), mdx(\"pre\", null, mdx(\"code\", _extends({\n    parentName: \"pre\"\n  }, {\n    \"className\": \"language-js\"\n  }), \"import { TextLoader } from '@langchain/classic/document_loaders/fs/text';\\n\\nconst loader = new TextLoader('./notes/pgvector.txt');\\nconst docs = await loader.load();\\n\\nconsole.log(docs[0].pageContent);\\nconsole.log(docs[0].metadata.source);\\n\")), mdx(\"p\", null, \"The loader sets \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"metadata.source\"), \" to the file path - keep it for citations in RAG answers.\"), mdx(\"h3\", {\n    \"id\": \"load-a-directory\"\n  }, \"Load a directory\"), mdx(\"p\", null, \"Use \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"DirectoryLoader\"), \" when you have many files. Map extensions to loader factories:\"), mdx(\"pre\", null, mdx(\"code\", _extends({\n    parentName: \"pre\"\n  }, {\n    \"className\": \"language-js\"\n  }), \"import { DirectoryLoader } from '@langchain/classic/document_loaders/fs/directory';\\nimport { TextLoader } from '@langchain/classic/document_loaders/fs/text';\\n\\nconst loader = new DirectoryLoader('./notes', {\\n  '.txt': (path) => new TextLoader(path),\\n  '.md': (path) => new TextLoader(path)\\n});\\n\\nconst docs = await loader.load();\\nconsole.log(`Loaded ${docs.length} documents`);\\n\")), mdx(\"p\", null, \"PDF, CSV, and JSON loaders are available via other integration packages. This post uses \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \".txt\"), \" and \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \".md\"), \" files.\"), mdx(\"h3\", {\n    \"id\": \"split-documents\"\n  }, \"Split documents\"), mdx(\"p\", null, \"Chunking makes retrieval more precise. Instead of embedding one large file, split it into smaller overlapping parts. Pass the \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"docs\"), \" array from \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"TextLoader\"), \" or \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"DirectoryLoader\"), \" to a splitter:\"), mdx(\"p\", null, \"Two parameters matter most:\"), mdx(\"ul\", null, mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"chunkSize\"), \" - target maximum size per chunk (characters or tokens, depending on splitter)\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"chunkOverlap\"), \" - shared text between adjacent chunks so context is not lost at boundaries\")), mdx(\"p\", null, \"Start with \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"chunkSize: 800\"), \" and \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"chunkOverlap: 120\"), \", then tune based on document style and answer quality.\"), mdx(\"pre\", null, mdx(\"code\", _extends({\n    parentName: \"pre\"\n  }, {\n    \"className\": \"language-js\"\n  }), \"import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\\n\\nconst splitter = new RecursiveCharacterTextSplitter({\\n  chunkSize: 800,\\n  chunkOverlap: 120\\n});\\n\\nconst chunks = await splitter.splitDocuments(docs);\\nconsole.log(chunks.length);\\n\")), mdx(\"h3\", {\n    \"id\": \"splitter-comparison\"\n  }, \"Splitter comparison\"), mdx(\"p\", null, \"The example above uses \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"RecursiveCharacterTextSplitter\"), \", the default for most RAG setups. Alternatives:\"), mdx(\"table\", null, mdx(\"thead\", {\n    parentName: \"table\"\n  }, mdx(\"tr\", {\n    parentName: \"thead\"\n  }, mdx(\"th\", _extends({\n    parentName: \"tr\"\n  }, {\n    \"align\": null\n  }), \"Splitter\"), mdx(\"th\", _extends({\n    parentName: \"tr\"\n  }, {\n    \"align\": null\n  }), \"Best for\"))), mdx(\"tbody\", {\n    parentName: \"table\"\n  }, mdx(\"tr\", {\n    parentName: \"tbody\"\n  }, mdx(\"td\", _extends({\n    parentName: \"tr\"\n  }, {\n    \"align\": null\n  }), mdx(\"inlineCode\", {\n    parentName: \"td\"\n  }, \"RecursiveCharacterTextSplitter\")), mdx(\"td\", _extends({\n    parentName: \"tr\"\n  }, {\n    \"align\": null\n  }), \"Default choice; tries paragraphs, then sentences, then words\")), mdx(\"tr\", {\n    parentName: \"tbody\"\n  }, mdx(\"td\", _extends({\n    parentName: \"tr\"\n  }, {\n    \"align\": null\n  }), mdx(\"inlineCode\", {\n    parentName: \"td\"\n  }, \"CharacterTextSplitter\")), mdx(\"td\", _extends({\n    parentName: \"tr\"\n  }, {\n    \"align\": null\n  }), \"Fixed character windows when structure does not matter\")), mdx(\"tr\", {\n    parentName: \"tbody\"\n  }, mdx(\"td\", _extends({\n    parentName: \"tr\"\n  }, {\n    \"align\": null\n  }), mdx(\"inlineCode\", {\n    parentName: \"td\"\n  }, \"TokenTextSplitter\")), mdx(\"td\", _extends({\n    parentName: \"tr\"\n  }, {\n    \"align\": null\n  }), \"When chunk limits must match model token budgets\")))), mdx(\"p\", null, mdx(\"strong\", {\n    parentName: \"p\"\n  }, \"Character-based:\")), mdx(\"pre\", null, mdx(\"code\", _extends({\n    parentName: \"pre\"\n  }, {\n    \"className\": \"language-js\"\n  }), \"import { CharacterTextSplitter } from '@langchain/textsplitters';\\n\\nconst splitter = new CharacterTextSplitter({\\n  chunkSize: 800,\\n  chunkOverlap: 120\\n});\\n\\nconst chunks = await splitter.splitDocuments(docs);\\n\")), mdx(\"p\", null, mdx(\"strong\", {\n    parentName: \"p\"\n  }, \"Token-based:\")), mdx(\"pre\", null, mdx(\"code\", _extends({\n    parentName: \"pre\"\n  }, {\n    \"className\": \"language-js\"\n  }), \"import { TokenTextSplitter } from '@langchain/textsplitters';\\n\\nconst splitter = new TokenTextSplitter({\\n  encodingName: 'cl100k_base',\\n  chunkSize: 200,\\n  chunkOverlap: 20\\n});\\n\\nconst chunks = await splitter.splitDocuments(docs);\\n\")), mdx(\"p\", null, \"Use token-based splitting when chunks must fit within a model's context window. Character-based recursive splitting is the usual starting point for RAG over prose.\"), mdx(\"h3\", {\n    \"id\": \"metadata-through-the-pipeline\"\n  }, \"Metadata through the pipeline\"), mdx(\"p\", null, \"Pass metadata when creating documents manually, or rely on loader metadata - splitters preserve it on each chunk:\"), mdx(\"pre\", null, mdx(\"code\", _extends({\n    parentName: \"pre\"\n  }, {\n    \"className\": \"language-js\"\n  }), \"const splitter = new RecursiveCharacterTextSplitter({\\n  chunkSize: 400,\\n  chunkOverlap: 60\\n});\\n\\nconst chunks = await splitter.createDocuments(\\n  ['First paragraph.\\\\n\\\\nSecond paragraph.'],\\n  [{ source: 'manual', section: 'intro' }]\\n);\\n\\nconsole.log(chunks[0].metadata);\\n\")), mdx(\"p\", null, \"After \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"splitDocuments(docs)\"), \", each chunk keeps fields like \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"source\"), \" from the parent document. Use those fields when storing chunks in a vector database or displaying citations.\"), mdx(\"h3\", {\n    \"id\": \"choosing-parameters\"\n  }, \"Choosing parameters\"), mdx(\"ul\", null, mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"Short FAQs or API docs\"), \" - smaller \", mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"chunkSize\"), \" (300\\u2013500) for precise retrieval\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"Long guides or blog posts\"), \" - larger \", mdx(\"inlineCode\", {\n    parentName: \"li\"\n  }, \"chunkSize\"), \" (800\\u20131200) to keep sections together\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"More overlap\"), \" - helps when answers span chunk boundaries; increases storage and embedding cost\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"Less overlap\"), \" - fewer redundant chunks; risk losing context at splits\")), mdx(\"p\", null, \"Tune with real questions from your domain.\"), mdx(\"h3\", {\n    \"id\": \"demo\"\n  }, \"Demo\"), mdx(\"p\", null, \"Runnable loader and splitter scripts for this post live in the \", mdx(\"inlineCode\", {\n    parentName: \"p\"\n  }, \"langchain-loaders-chunking-demo\"), \" folder. Get access via \", mdx(\"a\", _extends({\n    parentName: \"p\"\n  }, {\n    \"href\": \"https://sevic.dev/demos\"\n  }), \"code demos\"), \".\"));\n}\n;\nMDXContent.isMDXComponent = true;"}},"pageContext":{"id":"d12dd98f-1f01-5a2a-ad8b-faa09a10956b"}},"staticQueryHashes":["1961101537","2542493696"]}