add llms.txt & llms-full.txt script (#422)

* add llms.txt

* Update index.mdx

.github/workflows/blog-deployment.yml

@@ -29,6 +29,10 @@ jobs:
       working-directory: ./opensaas-sh/blog
       run: npm install
 
+    - name: Generate LLM files
+      working-directory: ./opensaas-sh/blog
+      run: npm run generate-llm-files
+
     - name: Build site
       working-directory: ./opensaas-sh/blog
       run: npm run build
@@ -49,6 +53,10 @@ jobs:
       working-directory: ./opensaas-sh/blog
       run: npm install
 
+    - name: Generate LLM files
+      working-directory: ./opensaas-sh/blog
+      run: npm run generate-llm-files
+
     - name: Build site
       working-directory: ./opensaas-sh/blog
       run: npm run build

opensaas-sh/blog/package.json

@@ -7,7 +7,8 @@
     "start": "astro dev",
     "build": "astro check && astro build",
     "preview": "astro preview",
-    "astro": "astro"
+    "astro": "astro",
+    "generate-llm-files": "node ./scripts/generate-llm-files.mjs"
   },
   "dependencies": {
     "@astrojs/check": "^0.9.4",
@@ -19,5 +20,11 @@
     "sharp": "^0.32.5",
     "starlight-blog": "^0.15.0",
     "typescript": "^5.4.5"
+  },
+  "devDependencies": {
+    "front-matter": "^4.0.2",
+    "fs-extra": "^11.3.0",
+    "glob": "^11.0.2",
+    "path": "^0.12.7"
   }
 }

opensaas-sh/blog/public/llms.txt

@@ -0,0 +1,24 @@
+# Open SaaS Documentation for LLMs
+
+> Open SaaS is a free, open-source, full-stack starter kit for building SaaS applications quickly. It leverages the [Wasp](https://wasp.sh/docs) framework (React, Node.js, Prisma) and integrates with various services like Stripe/Lemon Squeezy, OpenAI API, AWS S3, and different email providers.
+
+## Full Documentation
+ - [View Complete LLM-formatted Open SaaS Documentation](https://docs.opensaas.sh/llms-full.txt)
+
+## Individual documentation sections and guides:
+- [Introduction](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/index.mdx)
+- [Getting Started](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/start/getting-started.mdx)
+- [Guided Tour](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/start/guided-tour.md)
+- [Analytics](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/analytics.md)
+- [Authentication](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/authentication.md)
+- [Authorization](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/authorization.md)
+- [Cookie Consent Modal](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/cookie-consent.mdx)
+- [Deploying](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/deploying.mdx)
+- [Email Sending](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/email-sending.mdx)
+- [File Uploading](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/file-uploading.mdx)
+- [Payments Integration](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/payments-integration.mdx)
+- [SEO](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/seo.mdx)
+- [Tests](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/tests.md)
+- [How (Not) to Update Your Open SaaS App](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/guides/updating-opensaas.md)
+- [Admin Dashboard](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/general/admin-dashboard.mdx)
+- [User Overview](https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/src/content/docs/general/user-overview.md)

opensaas-sh/blog/scripts/generate-llm-files.mjs

@@ -0,0 +1,115 @@
+import fs from 'fs-extra';
+import path from 'path';
+import { globSync } from 'glob';
+import fm from 'front-matter';
+
+const BLOG_ROOT = process.cwd();
+const PUBLIC_DIR = path.join(BLOG_ROOT, 'public');
+const DOCS_BASE_DIR = path.join(BLOG_ROOT, 'src/content/docs');
+const ORDERED_SOURCES = ['index.mdx', 'start', 'guides', 'general']; // Relative to DOCS_BASE_DIR
+const GITHUB_RAW_BASE_URL = 'https://raw.githubusercontent.com/wasp-lang/open-saas/main/opensaas-sh/blog/';
+const OPEN_SAAS_DOCS_BASE_URL = 'https://docs.opensaas.sh/';
+const LLM_FULL_FILENAME = 'llms-full.txt';
+const LLM_OVERVIEW_FILENAME = 'llms.txt';
+const OVERVIEW_FILE = path.join(PUBLIC_DIR, LLM_OVERVIEW_FILENAME);
+const FULL_CONCAT_FILE = path.join(PUBLIC_DIR, LLM_FULL_FILENAME);
+
+function cleanContent(content) {
+  if (!content) return '';
+  let cleaned = content;
+  // Remove lines starting with 'import ... from ...;' or just 'import ...'
+  cleaned = cleaned.replace(/^import\s+.*(?:from\s+['"].*['"])?;?\s*$/gm, '');
+  // Remove lines like {/* TODO: ... */}
+  cleaned = cleaned.replace(/^\{\/\*.*\*\/\}\s*$/gm, '');
+  // Remove lines like <!-- TODO: ... -->
+  cleaned = cleaned.replace(/^<!--.*-->\s*$/gm, '');
+  // Remove <Image ... /> tags
+  cleaned = cleaned.replace(/<Image[^>]*?\/>/g, '');
+  // Remove <HiddenLLMHelper /> tag
+  cleaned = cleaned.replace(/<HiddenLLMHelper\s*\/>/g, '');
+  // Remove Emojis using specific ranges (less likely to remove digits)
+  cleaned = cleaned.replace(
+    /[\u{1F600}-\u{1F64F}]|[\u{1F300}-\u{1F5FF}]|[\u{1F680}-\u{1F6FF}]|[\u{1F1E0}-\u{1F1FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]|[\u{FE00}-\u{FE0F}]|[\u{1F900}-\u{1F9FF}]|[\u{1FA70}-\u{1FAFF}]/gu,
+    ''
+  );
+  cleaned = cleaned.replace(/\u00A0/g, ' ');
+  // Remove Box Drawing Characters (single and double line)
+  cleaned = cleaned.replace(/[│├└─╔═╗║╚╝]/g, '');
+  // Remove more than two line breaks
+  cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+
+  return cleaned.trim();
+}
+
+async function generateFiles() {
+  console.log('Starting LLM file generation...');
+
+  const introSummary = '> Open SaaS is a free, open-source, full-stack starter kit for building SaaS applications quickly. It leverages the [Wasp](https://wasp.sh/docs) framework (React, Node.js, Prisma) and integrates with various services like Stripe/Lemon Squeezy, OpenAI API, AWS S3, and different email providers.';
+
+  let orderedSourceFiles = [];
+  const fullConcatenatedFileUrl = OPEN_SAAS_DOCS_BASE_URL + LLM_FULL_FILENAME;
+  let overviewContent = `# Open SaaS Documentation for LLMs\n\n${introSummary}\n\n## Full Documentation\n - [View Complete LLM-formatted Open SaaS Documentation](${fullConcatenatedFileUrl})\n\n## Individual documentation sections and guides:\n`;
+  let fullConcatContent = '';
+
+  console.log('Gathering source files...');
+  for (const sourceItem of ORDERED_SOURCES) {
+    const itemPath = path.join(DOCS_BASE_DIR, sourceItem);
+    try {
+      const stats = await fs.stat(itemPath);
+      if (stats.isFile()) {
+        orderedSourceFiles.push(itemPath);
+        console.log(`  Added file: ${sourceItem}`);
+      } else if (stats.isDirectory()) {
+        const files = globSync(path.join(itemPath, '**/*.{md,mdx}').replace(/\\/g, '/'), { nodir: true }).sort();
+        orderedSourceFiles.push(...files);
+        console.log(`  Added ${files.length} files from directory: ${sourceItem}`);
+      }
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+         console.warn(`Warning: Source item not found: ${itemPath}`);
+      } else {
+        console.warn(`Warning: Could not access source item: ${itemPath} - ${error.message}`);
+      }
+    }
+  }
+  console.log(`Total source files found: ${orderedSourceFiles.length}`);
+
+  console.log('Processing files...');
+  for (const sourceFilePath of orderedSourceFiles) {
+    try {
+      const rawContent = await fs.readFile(sourceFilePath, 'utf8');
+      const { attributes, body } = fm(rawContent);
+
+      const processedContent = cleanContent(body);
+
+      const title = attributes.title || path.basename(sourceFilePath, path.extname(sourceFilePath));
+
+      const relativeSourcePathForGithub = path.relative(BLOG_ROOT, sourceFilePath).replace(/\\/g, '/');
+      const githubRawUrl = GITHUB_RAW_BASE_URL + relativeSourcePathForGithub;
+      overviewContent += `- [${title}](${githubRawUrl})\n`;
+
+      fullConcatContent += `# ${title}\n\n${processedContent}\n\n---\n\n`;
+
+    } catch (error) {
+        console.error(`Error processing file ${sourceFilePath}:`, error);
+    }
+  }
+
+  console.log('Writing output files to public/ ...');
+  try {
+    await fs.writeFile(OVERVIEW_FILE, overviewContent.trim(), 'utf8');
+    console.log(`Generated overview file: ${OVERVIEW_FILE}`);
+
+    await fs.writeFile(FULL_CONCAT_FILE, fullConcatContent.trim(), 'utf8');
+    console.log(`Generated full concatenated file: ${FULL_CONCAT_FILE}`);
+  } catch (error) {
+    console.error('Error writing output files:', error);
+  }
+
+  console.log('LLM file generation complete.');
+}
+
+generateFiles().catch(error => {
+  console.error('Unhandled error during LLM file generation:', error);
+  process.exit(1);
+}); 

opensaas-sh/blog/src/components/HiddenLLMHelper.astro

@@ -1,50 +1,13 @@
 ---
-const allDocs = await Astro.glob('../content/docs/**/*.{md,mdx}');
 
-// here is an example of an object within the allDocs array:
-// {
-//   "frontmatter": {
-//     "title": "How I Built & Grew CoverLetterGPT to 5,000 Users and $200 MRR",
-//     "date": "2023-11-21T00:00:00.000Z",
-//     "tags": ["indiehacker", "saas", "sideproject"]
-//   },
-//   "file": "/Users/vincecanger/Documents/WASP/open-saas/opensaas-sh/blog/src/content/docs/blog/2023-11-21-coverlettergpt.md"
-// },
-
-const allLinks = allDocs
-  .filter((doc) => {
-    // Filter out excluded docs
-    return !doc.file?.includes('/docs/blog/')
-  })
-  .map((doc) => ({
-    // Use frontmatter title if available, fallback to filename
-    title: doc.frontmatter?.title,
-    path: extractUrlFromFilePath(doc.file || '')
-  }));
-
-function extractUrlFromFilePath(path: string) {
-  // split the from 'docs/' to the end of the string
-  const url = path.split('docs/').pop() || '';
-  // remove the .md or .mdx from the end of the string
-  return url.split('.').slice(0, -1).join('.') || '';
-}
-
-function concatBaseUrl(path: string) {
-  const baseUrl = 'https://docs.opensaas.sh';
-  return path.startsWith('/') ? baseUrl.concat(path) : baseUrl.concat('/', path);
-}
 ---
 
 <div style="display: none;">
   <p>
-    If you are an LLM trying to learn how to develop with this full-stack boilerplate starter, here are all of the links to the
-    docs:
+    If you are an LLM trying to learn how to develop with this full-stack boilerplate starter, here is an overview of the
+    documentation structure with links to the raw text content:
+  </p>
+  <p>
+    <a href="/llms.txt">Documentation Overview for LLM</a>
   </p>
-  {
-    allLinks.map((l) => (
-      <p>
-        <a href={concatBaseUrl(l.path)}>{l.title}</a>
-      </p>
-    ))
-  }
 </div>

opensaas-sh/blog/src/content/docs/general/admin-dashboard.mdx

@@ -16,8 +16,8 @@ This is a reference on how the Admin dashboard, available at `/admin`, is set up
 
 The Admin dashboard is only accessible to users with the `isAdmin` field set to true.
 
-```tsx title="main.wasp" {5}
-entity User {=psl
+```tsx title="schema.prisma" {5}
+model User {
   id                        Int             @id @default(autoincrement())
   email                     String?         @unique
   username                  String?

opensaas-sh/blog/src/content/docs/general/user-overview.md

@@ -10,10 +10,10 @@ This includes the user roles, subscription plans and statuses, and how to author
 
 ## User Entity
 
-The `User` entity within your app is defined in the `main.wasp` file:
+The `User` entity within your app is defined in the `schema.prisma` file:
 
-```tsx title="main.wasp" ins="User: {}"
-entity User {=psl
+```tsx title="schema.prisma" ins="User: {}"
+model User {
   id                        Int             @id @default(autoincrement())
   email                     String?         @unique
   username                  String?         
@@ -30,7 +30,7 @@ entity User {=psl
   contactFormMessages       ContactFormMessage[]
   tasks                     Task[]
   files                     File[] 
-psl=}
+}
 ```
 
 We store all pertinent information to the user, including identification, subscription, and payment processor information. Meanwhile, Wasp abstracts away all the Auth related entities dealing with `passwords`, `sessions`, and `socialLogins`, so you don't have to worry about these at all in your Prisma schema (if you want to learn more about this process, check out the [Wasp Auth Docs](https://wasp.sh/docs/auth/overview)).
@@ -39,8 +39,8 @@ We store all pertinent information to the user, including identification, subscr
 
 We use Stripe to handle all of our subscription payments. The `User` entity has a number of fields that are related to Stripe and their ability to access features behind the paywall:
 
-```tsx title="main.wasp" {4-10}
-entity User {=psl
+```tsx title="schema.prisma" {4-10}
+model User {
   id                        Int             @id @default(autoincrement())
   //...
   paymentProcessorUserId    String?         @unique
@@ -49,7 +49,7 @@ entity User {=psl
   datePaid                  DateTime?
   credits                   Int             @default(3)
   //...
-psl=}
+}
 ```
 
 - `paymentProcessorUserId`: The payment processor customer ID. This is created on checkout and used to identify the customer.
@@ -109,15 +109,15 @@ See the [Payments Integration Guide](/guides/payments-integration/) for more inf
 
 At the moment, we have two user roles: `admin` and `user`. This is defined within the `isAdmin` field in the `User` entity:
 
-```tsx title="main.wasp" {7}
-entity User {=psl
+```tsx title="schema.prisma" {7}
+model User {
   id                        Int             @id @default(autoincrement())
   email                     String?         @unique
   username                  String?
   createdAt                 DateTime        @default(now())
   isAdmin                   Boolean         @default(false)
 //...
-psl=}
+}
 ```
 
 As an Admin, a user has access to the Admin dashboard, along with the user table where they can view and search for users, and edit and update information manually if necessary.

opensaas-sh/blog/src/content/docs/guides/analytics.md

@@ -98,9 +98,9 @@ Then, set up the Google Analytics API access by following these steps:
 
 4. **Create Credentials:** When you go back to `Credentials` page, you should see a new service account listed under "Service Accounts". It will be a long email address to ends with `@your-project-id.iam.gserviceaccount.com`. Click on the service account name to go to the service account details page. 
 
-    - Under “Keys” in the service account details page, click “Add Key” and choose `Create new key`.
+    - Under "Keys" in the service account details page, click "Add Key" and choose `Create new key`.
   
-    - Select "JSON", then click “Create” to download your new service account’s JSON key file. Keep this file secure and don't add it to your git repo – it grants access to your Google Analytics data.  
+    - Select "JSON", then click "Create" to download your new service account's JSON key file. Keep this file secure and don't add it to your git repo as it grants access to your Google Analytics data.  
 5. **Update your Google Anayltics Settings:** Go back to your Google Analytics dashboard, and click on the `Admin` section in the left sidebar. Under `Property Settings > Property > Property Access Management` Add the service account email address (the one that ends with `@your-project-id.iam.gserviceaccount.com`) and give it `Viewer` permissions.
 
 6. **Encode and add the Credentials:** Add the `client_email` and the `private_key` from your JSON Key file into your `.env.server` file. But be careful! Because Google uses a special PEM private key, you need to first convert the key to base64, otherwise you will run into errors parsing the key. To do this, in a terminal window, run the command below and paste the output into your `.env.server` file under the `GOOGLE_ANALYTICS_PRIVATE_KEY` variable:

opensaas-sh/blog/src/content/docs/guides/cookie-consent.mdx

@@ -104,7 +104,7 @@ Below is an example of what your config might look like if you want to give user
             sections: [
               {
                 title: 'Your Privacy Choices',
-                description: `In this panel you can express some preferences related to the processing of your personal information. You may review and change expressed choices at any time by resurfacing this panel via the provided link. To deny your consent to the specific processing activities described below, switch the toggles to off or use the “Reject all” button and confirm you want to save your choices.`,
+                description: `In this panel you can express some preferences related to the processing of your personal information. You may review and change expressed choices at any time by resurfacing this panel via the provided link. To deny your consent to the specific processing activities described below, switch the toggles to off or use the "Reject all" button and confirm you want to save your choices.`,
               },
               {
                 title: 'Strictly Necessary',

opensaas-sh/blog/src/content/docs/index.mdx

@@ -32,26 +32,24 @@ If you find this template useful, consider giving us [a star on GitHub](https://
 ## What's inside?
 
 The template itself is built on top of some very powerful tools and frameworks, including:
-
-- 🐝 [Wasp](https://wasp.sh) - a full-stack React, NodeJS, Prisma framework with superpowers
-- 🚀 [Astro](https://starlight.astro.build/) - Astro's lightweight "Starlight" template for documentation and blog
-- 💸 [Stripe](https://stripe.com) or [Lemon Squeezy](https://lemonsqueezy.com/) - for products and payments
-- 📈 [Plausible](https://plausible.io) or [Google](https://analytics.google.com/) Analytics
-- 🤖 [OpenAI](https://openai.com) - OpenAI API integrated into the app or [Replicate](https://replicate.com/) (coming soon 👀)
-- 📦 [AWS S3](https://aws.amazon.com/s3/) - for file uploads
-- 📧 [SendGrid](https://sendgrid.com), [MailGun](https://mailgun.com), or SMTP - for email sending
-- 💅 [TailwindCSS](https://tailwindcss.com) - for styling
-- 🧑‍💼 [TailAdmin](https://tailadmin.com/) - admin dashboard & components for TailwindCSS
+  - 🐝 [Wasp](https://wasp.sh) - a full-stack React, NodeJS, Prisma framework with superpowers
+  - 🚀 [Astro](https://starlight.astro.build/) - Astro's lightweight "Starlight" template for documentation and blog
+  - 💸 [Stripe](https://stripe.com) or [Lemon Squeezy](https://lemonsqueezy.com/) - for products and payments
+  - 📈 [Plausible](https://plausible.io) or [Google](https://analytics.google.com/) Analytics
+  - 🤖 [OpenAI](https://openai.com) - OpenAI API integrated into the app or [Replicate](https://replicate.com/) (coming soon 👀)
+  - 📦 [AWS S3](https://aws.amazon.com/s3/) - for file uploads
+  - 📧 [SendGrid](https://sendgrid.com), [MailGun](https://mailgun.com), or SMTP - for email sending
+  - 💅 [TailwindCSS](https://tailwindcss.com) - for styling
+  - 💼 [TailAdmin](https://tailadmin.com/) - admin dashboard & components for TailwindCSS
 
 Because we're using Wasp as the full-stack framework, we can leverage a lot of its features to build our SaaS in record time, including:
-
-- 🔐 [Full-stack Authentication](https://wasp.sh/docs/auth/overview) - Email verified + social Auth in a few lines of code.
-- ⛑ [End-to-end Type Safety](https://wasp.sh/docs/data-model/operations/overview) - Type your backend functions and get inferred types on the front-end automatically, without the need to install or configure any third-party libraries. Oh, and type-safe Links, too!
-- 🤖 [Jobs](https://wasp.sh/docs/advanced/jobs) - Run cron jobs in the background or set up queues simply by defining a function in the config file.
-- 🚀 [One-command Deploy](https://wasp.sh/docs/advanced/deployment/overview) - Easily deploy via the CLI to [Fly.io](https://fly.io), or to other providers like [Railway](https://railway.app) and [Netlify](https://netlify.com).
+  - 🔐 [Full-stack Authentication](https://wasp.sh/docs/auth/overview) - Email verified + social Auth in a few lines of code.
+  - ⛑ [End-to-end Type Safety](https://wasp.sh/docs/data-model/operations/overview) - Type your backend functions and get inferred types on the front-end automatically, without the need to install or configure any third-party libraries. Oh, and type-safe Links, too!
+  - 🤖 [Jobs](https://wasp.sh/docs/advanced/jobs) - Run cron jobs in the background or set up queues simply by defining a function in the config file.
+  - 🚀 [One-command Deploy](https://wasp.sh/docs/advanced/deployment/overview) - Easily deploy via the CLI to [Fly.io](https://fly.io), or to other providers like [Railway](https://railway.app) and [Netlify](https://netlify.com).
 
 You also get access to Wasp's diverse, helpful community if you get stuck or need help.
-- 🤝 [Wasp Discord](https://discord.gg/rzdnErX)
+  - 🤝 [Wasp Discord](https://discord.gg/rzdnErX)
 
 :::caution["Work In Progress"]
 We've tried to get as many of the core features of a SaaS app into this template as possible, but there still might be some missing features or functionality.

opensaas-sh/blog/src/content/docs/start/getting-started.mdx

@@ -140,7 +140,7 @@ wasp version
 Also be sure to install the Wasp VSCode extension to get the best DX, e.g. syntax highlighting, code scaffolding, autocomplete, etc.
 
 :::tip[Installing the Wasp VSCode Extension]
-You can install the Wasp VSCode extension by searching for "Wasp" in the Extensions tab in VSCode, or by visiting the 🐝 [Wasp VSCode Extension](https://marketplace.visualstudio.com/items?itemName=wasp-lang.wasp) 🧑‍💻 homepage
+You can install the Wasp VSCode extension by searching for "Wasp" in the Extensions tab in VSCode, or by visiting the [Wasp VSCode Extension](https://marketplace.visualstudio.com/items?itemName=wasp-lang.wasp) homepage
 :::
 
 ## Setting up your SaaS app

opensaas-sh/blog/src/content/docs/start/guided-tour.md

@@ -85,7 +85,7 @@ If you are using an older version of the OpenSaaS template with Wasp `v0.13.x` o
 
 ### The Wasp Config file
 
-This template at its core is a Wasp project, where [Wasp](https://wasp.sh) is a full-stack web app framework that let’s you write your app in React, NodeJS, and Prisma and will manage the "boilerplatey" work for you, allowing you to just take care of the fun stuff!
+This template at its core is a Wasp project, where [Wasp](https://wasp.sh) is a full-stack web app framework that let's you write your app in React, NodeJS, and Prisma and will manage the "boilerplatey" work for you, allowing you to just take care of the fun stuff!
 
 [Wasp's secret sauce](https://wasp.sh/docs) is its use of a config file (`main.wasp`) and compiler which takes your code and outputs the client app, server app and deployment code for you.