Feat: Caching and optional indexing of site.pages

Signed-off-by: Karthik Ganeshram <karthik.ganeshram@fermyon.com>

.gitignore

@@ -2,3 +2,4 @@
 /modules/*.wasm
 /logs
 node_modules/
+/config/_cache.json

Makefile

@@ -2,6 +2,7 @@ SPIN ?= spin
 # If PREVIEW_MODE is on then unpublished content will be displayed.
 PREVIEW_MODE ?= 0
 SHOW_DEBUG ?= 1
+DISABLE_CACHE ?= 1
 BASE_URL ?= http://localhost:3000
 
 .PHONY: build
@@ -31,7 +32,11 @@ check-content:
 .PHONY: serve
 serve: build
 serve:
-	$(SPIN) up --log-dir ./logs -e PREVIEW_MODE=$(PREVIEW_MODE) -e SHOW_DEBUG=$(SHOW_DEBUG) -e BASE_URL=$(BASE_URL) -f docs/spin.toml
+	$(SPIN) up --log-dir ./logs -e PREVIEW_MODE=$(PREVIEW_MODE) -e SHOW_DEBUG=$(SHOW_DEBUG) -e BASE_URL=$(BASE_URL) -e DISABLE_CACHE=$(DISABLE_CACHE) -f docs/spin.toml
 
 .PHONY: run
 run: serve
+
+.PHONY: clean
+clean:
+	rm -rf config/_cache.json

README.md

@@ -217,6 +217,15 @@ Make:
 $ PREVIEW_MODE=1 make serve
 ```
 
+### The Page Cache
+
+Unless the environment variable `-e DISABLE_CACHE=1` is set, the first load of a site will create a cache
+of page metadata in `config/_cache.json`. This is an optimization to reduce the number of file IO operations
+Bartholomew needs to make.
+
+If you are actively developing content, we suggest setting `DISABLE_CACHE=1`. By default, the `Makefile`'s `make serve`
+target disables the cache, as `make serve` is assumed to be used only for developers.
+
 ## Configuring Bartholomew
 
 Bartholomew can run inside of any Spin environment that supports directly executing

docs/content/configuration.md

@@ -17,7 +17,11 @@ title = "Bartholomew"
 base_url = "http://localhost:3000"
 about = "This site is generated with Bartholomew, the Spin micro-CMS. And this message is in site.toml."
 theme = "fermyon"
+<<<<<<< HEAD
+index_site_themes = ["main"]
+=======
 enable_shortcodes = false
+>>>>>>> main
 
 [extra]
 copyright = "The Site Authors"
@@ -34,7 +38,11 @@ It has a few pre-defined fields:
 - base_url: a base URL that templates can use to construct full URLs to content. This can be overridden by setting the `-e BASE_URL="https://example.com"` environment variable for Spin.
 - about: a brief description of the site
 - theme: the name of the theme for the website from the `/themes/` folder
+<<<<<<< HEAD
+- index_site_themes: A list of templates that require `site.pages` to be populated.
+=======
 - enable_shortcodes: Allows addition of shortcodes in the markdown content using rhai scripts. Defaults to false.
+>>>>>>> main
 
 You can define your own fields in the `[extra]` section. Anything in `[extra]` is not
 used by the system itself. But it's a useful way to pass information from one central

docs/content/templates.md

@@ -61,7 +61,7 @@ the `extra` section, we use `{{ page.head.extra.key }}`.
 
 ### The `site` object
 
-In addition to the `page` object, there is also a `site` object:
+In addition to the `page` object, there is also a `site` object. `site.pages` contains the `head` section and content of every page in the site. `site.pages` is only populated for templates included in `include_site_pages` in `site.toml` as described in the [configuration section](/configuration.md) 
 
 ```
 {
@@ -81,7 +81,7 @@ In addition to the `page` object, there is also a `site` object:
 ```
 
 Note that the `site.pages` array has access to every single document in the `content` folder.
-This part of the API may change in the future, as it does not scale terribly well.
+This part of the API may change in the future, as it does not scale terribly well. 
 
 ### The `env` object
 

src/bartholomew.rs

@@ -19,6 +19,13 @@ pub fn render(req: Request) -> Result<Response> {
         }
         _ => false,
     };
+    let disable_cache = match std::env::var("DISABLE_CACHE") {
+        Ok(val) if val == "1" => {
+            eprintln!("INFO: Bartholomew is running with DISABLE_CACHE=1");
+            true
+        }
+        _ => false,
+    };
 
     // Get the request path.
     let path_info = match req.headers().get("spin-path-info") {
@@ -61,6 +68,7 @@ pub fn render(req: Request) -> Result<Response> {
 
     // If running in preview mode, show unpublished content.
     engine.show_unpublished = preview_mode;
+    engine.disable_cache = disable_cache;
 
     // Load the template directory.
     engine.load_template_dir()?;

src/content.rs

@@ -12,6 +12,8 @@ use crate::template::PageValues;
 use handlebars::Handlebars;
 
 const DOC_SEPARATOR: &str = "\n---\n";
+// Cache for page values to reduce IO on each request
+const CACHE_FILE: &str = "/config/_cache.json";
 
 const SHORTCODE_PATH: &str = "/shortcodes/";
 
@@ -71,11 +73,38 @@ pub fn content_path(content_dir: PathBuf, path_info: &str) -> PathBuf {
     content_dir.join(buf.strip_prefix("/").unwrap_or(&buf))
 }
 
-/// Fetch all pages.
+pub fn all_pages(
+    dir: PathBuf,
+    show_unpublished: bool,
+    skip_cache: bool,
+) -> anyhow::Result<BTreeMap<String, PageValues>> {
+    if skip_cache {
+        return all_pages_load(dir, show_unpublished);
+    }
+
+    // Try loading the cached object:
+    let cache = PathBuf::from(CACHE_FILE);
+    match std::fs::read_to_string(&cache) {
+        Ok(data) => {
+            // We have the whole site here.
+            serde_json::from_str(&data)
+                .map_err(|e| anyhow::anyhow!("Failed to parse page cache TOML: {}", e))
+        }
+        Err(_) => {
+            let contents = all_pages_load(dir, show_unpublished)?;
+            // Serialize the files back out to disk for subsequent requests.
+            let cache_data = serde_json::to_string(&contents)?;
+            std::fs::write(&cache, cache_data)?;
+            Ok(contents)
+        }
+    }
+}
+
+/// Fetch all pages from disk.
 ///
 /// If show_unpublished is `true`, this will include pages that Bartholomew has determined are
 /// unpublished.
-pub fn all_pages(
+pub fn all_pages_load(
     dir: PathBuf,
     show_unpublished: bool,
 ) -> anyhow::Result<BTreeMap<String, PageValues>> {

src/template.rs

@@ -20,6 +20,7 @@ pub struct SiteInfo {
     pub base_url: Option<String>,
     pub about: Option<String>,
     pub theme: Option<String>,
+    pub index_site_pages: Option<Vec<String>>,
     pub extra: BTreeMap<String, String>,
 }
 
@@ -53,7 +54,7 @@ pub struct SiteValues {
 
 /// The structured values sent to the template renderer.
 /// The body should be legal HTML that can be inserted within the <body> tag.
-#[derive(Serialize)]
+#[derive(Serialize, Deserialize)]
 pub struct PageValues {
     pub head: Head,
     pub body: String,
@@ -77,6 +78,7 @@ pub struct Renderer<'a> {
     pub script_dir: PathBuf,
     pub content_dir: PathBuf,
     pub show_unpublished: bool,
+    pub disable_cache: bool,
     handlebars: handlebars::Handlebars<'a>,
 }
 
@@ -94,6 +96,7 @@ impl<'a> Renderer<'a> {
             script_dir,
             content_dir,
             show_unpublished: false,
+            disable_cache: false,
             handlebars: Handlebars::new(),
         }
     }
@@ -108,7 +111,7 @@ impl<'a> Renderer<'a> {
         self.register_helpers();
 
         // If there is a theme, load the templates provided by it first
-        // Allows for user defined tempaltes to take precedence
+        // Allows for user defined templates to take precedence
         if self.theme_dir.is_some() {
             let mut templates = self.theme_dir.as_ref().unwrap().to_owned();
             templates.push("templates");
@@ -175,7 +178,6 @@ impl<'a> Renderer<'a> {
             page,
             request: request_headers,
             site: SiteValues {
-                info,
                 // Right now, we literally include ALL OF THE CONTENT in its rendered
                 // state. I take some consolation in knowing how PHP works. But
                 // seriously, this is probably not the best thing to do.
@@ -186,7 +188,21 @@ impl<'a> Renderer<'a> {
                 // 3. ???
                 // 4. Leave it like it is
                 // 5. Determine that this is out of scope
-                pages: crate::content::all_pages(self.content_dir.clone(), self.show_unpublished)?,
+                pages: match &info.index_site_pages {
+                    Some(templates) => {
+                        if templates.contains(&tpl) {
+                            crate::content::all_pages(
+                                self.content_dir.clone(),
+                                self.show_unpublished,
+                                self.disable_cache,
+                            )?
+                        } else {
+                            BTreeMap::new()
+                        }
+                    }
+                    None => BTreeMap::new(),
+                },
+                info,
             },
             // Copy the WASI env into the env template var.
             env: std::env::vars().collect(),