highperfocused/open-saas
1
0
Fork 0
mirror of https://github.com/wasp-lang/open-saas.git synced 2025-11-20 22:57:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

format

Browse Source
This commit is contained in:
Franjo Mindek
2025-09-18 11:42:49 +02:00
parent 054c1f8dc5
commit 3f463fb202
155 changed files with 4138 additions and 2994 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
opensaas-sh/blog/scripts/generate-llm-files.mjs
View File

@@ -1,57 +1,59 @@
import fs from 'fs-extra';
import path from 'path';
import { globSync } from 'glob';
import fm from 'front-matter';
import fm from "front-matter";
import fs from "fs-extra";
import { globSync } from "glob";
import path from "path";
const BLOG_ROOT = process.cwd();
const PUBLIC_DIR = path.join(BLOG_ROOT, 'public');
const DOCS_BASE_DIR = path.join(BLOG_ROOT, 'src/content/docs');
const ORDERED_SOURCES = ['index.mdx', 'start', 'guides', 'general']; // Relative to DOCS_BASE_DIR
const GITHUB_RAW_BASE_URL = 'https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/';
const OPEN_SAAS_DOCS_BASE_URL = 'https://docs.opensaas.sh/';
const LLM_FULL_FILENAME = 'llms-full.txt';
const LLM_OVERVIEW_FILENAME = 'llms.txt';
const PUBLIC_DIR = path.join(BLOG_ROOT, "public");
const DOCS_BASE_DIR = path.join(BLOG_ROOT, "src/content/docs");
const ORDERED_SOURCES = ["index.mdx", "start", "guides", "general"]; // Relative to DOCS_BASE_DIR
const GITHUB_RAW_BASE_URL =
"https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/";
const OPEN_SAAS_DOCS_BASE_URL = "https://docs.opensaas.sh/";
const LLM_FULL_FILENAME = "llms-full.txt";
const LLM_OVERVIEW_FILENAME = "llms.txt";
const OVERVIEW_FILE = path.join(PUBLIC_DIR, LLM_OVERVIEW_FILENAME);
const FULL_CONCAT_FILE = path.join(PUBLIC_DIR, LLM_FULL_FILENAME);
function cleanContent(content) {
if (!content) return '';
if (!content) return "";
let cleaned = content;
// Remove lines starting with 'import ... from ...;' or just 'import ...'
cleaned = cleaned.replace(/^import\s+.*(?:from\s+['"].*['"])?;?\s*$/gm, '');
cleaned = cleaned.replace(/^import\s+.*(?:from\s+['"].*['"])?;?\s*$/gm, "");
// Remove lines like {/* TODO: ... */}
cleaned = cleaned.replace(/^\{\/\*.*\*\/\}\s*$/gm, '');
cleaned = cleaned.replace(/^\{\/\*.*\*\/\}\s*$/gm, "");
// Remove lines like <!-- TODO: ... -->
cleaned = cleaned.replace(/^<!--.*-->\s*$/gm, '');
cleaned = cleaned.replace(/^<!--.*-->\s*$/gm, "");
// Remove <Image ... /> tags
cleaned = cleaned.replace(/<Image[^>]*?\/>/g, '');
cleaned = cleaned.replace(/<Image[^>]*?\/>/g, "");
// Remove <HiddenLLMHelper /> tag
cleaned = cleaned.replace(/<HiddenLLMHelper\s*\/>/g, '');
cleaned = cleaned.replace(/<HiddenLLMHelper\s*\/>/g, "");
// Remove Emojis using specific ranges (less likely to remove digits)
cleaned = cleaned.replace(
/[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]|[\u{FE00}-\u{FE0F}]|[\u{1F900}-\u{1F9FF}]|[\u{1FA70}-\u{1FAFF}]/gu,
''
"",
);
cleaned = cleaned.replace(/\u00A0/g, ' ');
cleaned = cleaned.replace(/\u00A0/g, " ");
// Remove Box Drawing Characters (single and double line)
cleaned = cleaned.replace(/[│├└─╔═╗║╚╝]/g, '');
cleaned = cleaned.replace(/[│├└─╔═╗║╚╝]/g, "");
// Remove more than two line breaks
cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
cleaned = cleaned.replace(/\n{3,}/g, "\n\n");
return cleaned.trim();
}
async function generateFiles() {
console.log('Starting LLM file generation...');
console.log("Starting LLM file generation...");
const introSummary = '> Open SaaS is a free, open-source, full-stack starter kit for building SaaS applications quickly. It leverages the [Wasp](https://wasp.sh/docs) framework (React, Node.js, Prisma) and integrates with various services like Stripe/Lemon Squeezy, OpenAI API, AWS S3, and different email providers.';
const introSummary =
"> Open SaaS is a free, open-source, full-stack starter kit for building SaaS applications quickly. It leverages the [Wasp](https://wasp.sh/docs) framework (React, Node.js, Prisma) and integrates with various services like Stripe/Lemon Squeezy, OpenAI API, AWS S3, and different email providers.";
let orderedSourceFiles = [];
const fullConcatenatedFileUrl = OPEN_SAAS_DOCS_BASE_URL + LLM_FULL_FILENAME;
let overviewContent = `# Open SaaS Documentation for LLMs\n\n${introSummary}\n\n## Full Documentation\n - [View Complete LLM-formatted Open SaaS Documentation](${fullConcatenatedFileUrl})\n\n## Individual documentation sections and guides:\n`;
let fullConcatContent = '';
let fullConcatContent = "";
console.log('Gathering source files...');
console.log("Gathering source files...");
for (const sourceItem of ORDERED_SOURCES) {
const itemPath = path.join(DOCS_BASE_DIR, sourceItem);
try {
@@ -60,56 +62,66 @@ async function generateFiles() {
orderedSourceFiles.push(itemPath);
console.log(` Added file: ${sourceItem}`);
} else if (stats.isDirectory()) {
const files = globSync(path.join(itemPath, '**/*.{md,mdx}').replace(/\\/g, '/'), { nodir: true }).sort();
const files = globSync(
path.join(itemPath, "**/*.{md,mdx}").replace(/\\/g, "/"),
{ nodir: true },
).sort();
orderedSourceFiles.push(...files);
console.log(` Added ${files.length} files from directory: ${sourceItem}`);
console.log(
` Added ${files.length} files from directory: ${sourceItem}`,
);
}
} catch (error) {
if (error.code === 'ENOENT') {
console.warn(`Warning: Source item not found: ${itemPath}`);
if (error.code === "ENOENT") {
console.warn(`Warning: Source item not found: ${itemPath}`);
} else {
console.warn(`Warning: Could not access source item: ${itemPath} - ${error.message}`);
console.warn(
`Warning: Could not access source item: ${itemPath} - ${error.message}`,
);
}
}
}
console.log(`Total source files found: ${orderedSourceFiles.length}`);
console.log('Processing files...');
console.log("Processing files...");
for (const sourceFilePath of orderedSourceFiles) {
try {
const rawContent = await fs.readFile(sourceFilePath, 'utf8');
const rawContent = await fs.readFile(sourceFilePath, "utf8");
const { attributes, body } = fm(rawContent);
const processedContent = cleanContent(body);
const title = attributes.title || path.basename(sourceFilePath, path.extname(sourceFilePath));
const title =
attributes.title ||
path.basename(sourceFilePath, path.extname(sourceFilePath));
const relativeSourcePathForGithub = path.relative(BLOG_ROOT, sourceFilePath).replace(/\\/g, '/');
const relativeSourcePathForGithub = path
.relative(BLOG_ROOT, sourceFilePath)
.replace(/\\/g, "/");
const githubRawUrl = GITHUB_RAW_BASE_URL + relativeSourcePathForGithub;
overviewContent += `- [${title}](${githubRawUrl})\n`;
fullConcatContent += `# ${title}\n\n${processedContent}\n\n---\n\n`;
} catch (error) {
console.error(`Error processing file ${sourceFilePath}:`, error);
console.error(`Error processing file ${sourceFilePath}:`, error);
}
}
console.log('Writing output files to public/ ...');
console.log("Writing output files to public/ ...");
try {
await fs.writeFile(OVERVIEW_FILE, overviewContent.trim(), 'utf8');
await fs.writeFile(OVERVIEW_FILE, overviewContent.trim(), "utf8");
console.log(`Generated overview file: ${OVERVIEW_FILE}`);
await fs.writeFile(FULL_CONCAT_FILE, fullConcatContent.trim(), 'utf8');
await fs.writeFile(FULL_CONCAT_FILE, fullConcatContent.trim(), "utf8");
console.log(`Generated full concatenated file: ${FULL_CONCAT_FILE}`);
} catch (error) {
console.error('Error writing output files:', error);
console.error("Error writing output files:", error);
}
console.log('LLM file generation complete.');
console.log("LLM file generation complete.");
}
generateFiles().catch(error => {
console.error('Unhandled error during LLM file generation:', error);
generateFiles().catch((error) => {
console.error("Unhandled error during LLM file generation:", error);
process.exit(1);
});
});