open-saas/opensaas-sh/blog/scripts/generate-llm-files.mjs

import fs from 'fs-extra';
import path from 'path';
import { globSync } from 'glob';
import fm from 'front-matter';

const BLOG_ROOT = process.cwd();
const PUBLIC_DIR = path.join(BLOG_ROOT, 'public');
const DOCS_BASE_DIR = path.join(BLOG_ROOT, 'src/content/docs');
const ORDERED_SOURCES = ['index.mdx', 'start', 'guides', 'general']; // Relative to DOCS_BASE_DIR
const GITHUB_RAW_BASE_URL = 'https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/';
const OPEN_SAAS_DOCS_BASE_URL = 'https://docs.opensaas.sh/';
const LLM_FULL_FILENAME = 'llms-full.txt';
const LLM_OVERVIEW_FILENAME = 'llms.txt';
const OVERVIEW_FILE = path.join(PUBLIC_DIR, LLM_OVERVIEW_FILENAME);
const FULL_CONCAT_FILE = path.join(PUBLIC_DIR, LLM_FULL_FILENAME);

function cleanContent(content) {
  if (!content) return '';
  let cleaned = content;
  // Remove lines starting with 'import ... from ...;' or just 'import ...'
  cleaned = cleaned.replace(/^import\s+.*(?:from\s+['"].*['"])?;?\s*$/gm, '');
  // Remove lines like {/* TODO: ... */}
  cleaned = cleaned.replace(/^\{\/\*.*\*\/\}\s*$/gm, '');
  // Remove lines like <!-- TODO: ... -->
  cleaned = cleaned.replace(/^<!--.*-->\s*$/gm, '');
  // Remove <Image ... /> tags
  cleaned = cleaned.replace(/<Image[^>]*?\/>/g, '');
  // Remove <HiddenLLMHelper /> tag
  cleaned = cleaned.replace(/<HiddenLLMHelper\s*\/>/g, '');
  // Remove Emojis using specific ranges (less likely to remove digits)
  cleaned = cleaned.replace(
    /[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]|[\u{FE00}-\u{FE0F}]|[\u{1F900}-\u{1F9FF}]|[\u{1FA70}-\u{1FAFF}]/gu,
    ''
  );
  cleaned = cleaned.replace(/\u00A0/g, ' ');
  // Remove Box Drawing Characters (single and double line)
  cleaned = cleaned.replace(/[│├└─╔═╗║╚╝]/g, '');
  // Remove more than two line breaks
  cleaned = cleaned.replace(/\n{3,}/g, '\n\n');

  return cleaned.trim();
}

async function generateFiles() {
  console.log('Starting LLM file generation...');

  const introSummary = '> Open SaaS is a free, open-source, full-stack starter kit for building SaaS applications quickly. It leverages the [Wasp](https://wasp.sh/docs) framework (React, Node.js, Prisma) and integrates with various services like Stripe/Lemon Squeezy, OpenAI API, AWS S3, and different email providers.';

  let orderedSourceFiles = [];
  const fullConcatenatedFileUrl = OPEN_SAAS_DOCS_BASE_URL + LLM_FULL_FILENAME;
  let overviewContent = `# Open SaaS Documentation for LLMs\n\n${introSummary}\n\n## Full Documentation\n - [View Complete LLM-formatted Open SaaS Documentation](${fullConcatenatedFileUrl})\n\n## Individual documentation sections and guides:\n`;
  let fullConcatContent = '';

  console.log('Gathering source files...');
  for (const sourceItem of ORDERED_SOURCES) {
    const itemPath = path.join(DOCS_BASE_DIR, sourceItem);
    try {
      const stats = await fs.stat(itemPath);
      if (stats.isFile()) {
        orderedSourceFiles.push(itemPath);
        console.log(`  Added file: ${sourceItem}`);
      } else if (stats.isDirectory()) {
        const files = globSync(path.join(itemPath, '**/*.{md,mdx}').replace(/\\/g, '/'), { nodir: true }).sort();
        orderedSourceFiles.push(...files);
        console.log(`  Added ${files.length} files from directory: ${sourceItem}`);
      }
    } catch (error) {
      if (error.code === 'ENOENT') {
         console.warn(`Warning: Source item not found: ${itemPath}`);
      } else {
        console.warn(`Warning: Could not access source item: ${itemPath} - ${error.message}`);
      }
    }
  }
  console.log(`Total source files found: ${orderedSourceFiles.length}`);

  console.log('Processing files...');
  for (const sourceFilePath of orderedSourceFiles) {
    try {
      const rawContent = await fs.readFile(sourceFilePath, 'utf8');
      const { attributes, body } = fm(rawContent);

      const processedContent = cleanContent(body);

      const title = attributes.title || path.basename(sourceFilePath, path.extname(sourceFilePath));

      const relativeSourcePathForGithub = path.relative(BLOG_ROOT, sourceFilePath).replace(/\\/g, '/');
      const githubRawUrl = GITHUB_RAW_BASE_URL + relativeSourcePathForGithub;
      overviewContent += `- [${title}](${githubRawUrl})\n`;

      fullConcatContent += `# ${title}\n\n${processedContent}\n\n---\n\n`;

    } catch (error) {
        console.error(`Error processing file ${sourceFilePath}:`, error);
    }
  }

  console.log('Writing output files to public/ ...');
  try {
    await fs.writeFile(OVERVIEW_FILE, overviewContent.trim(), 'utf8');
    console.log(`Generated overview file: ${OVERVIEW_FILE}`);

    await fs.writeFile(FULL_CONCAT_FILE, fullConcatContent.trim(), 'utf8');
    console.log(`Generated full concatenated file: ${FULL_CONCAT_FILE}`);
  } catch (error) {
    console.error('Error writing output files:', error);
  }

  console.log('LLM file generation complete.');
}

generateFiles().catch(error => {
  console.error('Unhandled error during LLM file generation:', error);
  process.exit(1);
});