major refactoring: remove 1-to-1 limit of template-to-output

Large refactoring that:
 * allow any template to be used any number of times
 * combine config's gitsy_templates and gitsy_outputs into gitsy_outputs
 * improves safety of output file names
 * prepares foundation for slugified permalinks
 * prepares foundation for parallelizing more execution

README.md

@@ -105,7 +105,7 @@ See the included `config.toml` for full documentation of the available settings.
 
 The top of the file contains global, site-wide settings like the site name, description, base URL, pagination rules, and memory limits.  Here you can also specify directories that contain Git repositories in subdirectories, which is used for bulk-import of many repositories.
 
-This should be followed by the `[gitsy_templates]` section and `[gitsy_outputs]` sections, which define the input templates and output paths respectively.  Input templates that are not specified will not be generated, so you can disable any output types that you don't need by commenting out the appropriate line in `[gitsy_templates]`
+This should be followed by the `[gitsy_outputs]` section, which defines which input templates to use, and which files to output.  Input templates that are not specified will not be generated, so you can disable any output types that you don't need.  Templates can be used as many times as desired, generating arbitrarily many outputs.
 
 An optional `[gitsy_extra]` section can be used to provide global, user-defined key/value pairs to all of the templates.  Use this if you want to add custom site-wide variables for use in your templates.
 

config.toml

@@ -298,185 +298,129 @@ syntax_highlight_theme = "base16-ocean.light"
 
 ###############################################################################
 ##
-## Subsection specifying which files to use as templates.
+## Subsection specifying output paths, and how they are generated.
 ##
-## The individual templates are relative to the `path` directory.  Each
-## template is a single file using the Tera template engine's format for text
-## substitution.
+## Itsy-Gitsy requires two root paths, both specified here:
 ##
-## All except `path` are optional.  If not specified, the associated outputs
-## will not be generated.
+## 1) `output_root`, a directory where all rendered output will be written
+## 2) `template_root`, a directory where all input templates are stored
 ##
-###############################################################################
-[gitsy_templates]
-
-# Path to a folder containing Tera templates.
-#
-# All files with a .html extension found under this directory, and its
-# immediate children directories, are imported into the Tera template engine.
-path = "templates/default_light/"
-
-# Template responsible for the list of repositories.
-#
-# This template is evaluated with the list of all configured repositories.  It
-# is intended for providing an overview of the available repositories, but the
-# full details of each repository are also included.
-#
-# This template executes one time.
-repo_list    = "repos.html"
-
-# Template responsible for summarizing a single repository.
-#
-# This template is evaluated with a single parsed repository, with all repo
-# data (commits, branches, etc) available.
-#
-# This template executes one time per repository.
-summary = "summary.html"
-
-# Template responsible for displaying the commit history.
-#
-# This template is evaluated with the same data as `summary`.  If the
-# `paginate_history` setting is non-zero, this may be called several
-# times with the `history` template variable reduced to the requested
-# page size, and with a `page` template variable provided to identify
-# the current, previous, and next pages.
-#
-# This template executes at least one time per repository, or several
-# times if paginated.
-history      = "history.html"
-
-# Template responsible for displaying a single commit.
-#
-# Called once per parsed commit, with both the whole repository and the current
-# commit available to the template.
-#
-# This template executes many times.
-commit       = "commit.html"
-
-# Template responsible for displaying the repo branches.
-#
-# This template is evaluated with the same data as `summary`.  If the
-# `paginate_branches` setting is non-zero, this may be called several
-# times with the `branches` template variable reduced to the requested
-# page size, and with a `page` template variable provided to identify
-# the current, previous, and next pages.
-#
-# This template executes at least one time per repository, or several
-# times if paginated.
-branches     = "branches.html"
-
-# Template responsible for displaying a single branch.
-#
-# Called once per parsed branch, with both the whole repository and the current
-# branch available to the template.
-#
-# This template executes many times.
-branch       = "branch.html"
-
-# Template responsible for displaying the repo tags.
-#
-# This template is evaluated with the same data as `summary`.  If the
-# `paginate_tags` setting is non-zero, this may be called several
-# times with the `tags` template variable reduced to the requested
-# page size, and with a `page` template variable provided to identify
-# the current, previous, and next pages.
-#
-# This template executes at least one time per repository, or several
-# times if paginated.
-tags         = "tags.html"
-
-# Template responsible for displaying a single tag.
-#
-# Called once per parsed tag, with both the whole repository and the current
-# tag available to the template.
-#
-# This template executes many times.
-tag          = "tag.html"
-
-# Template responsible for displaying file tree.
-#
-# Called with the same variables as the `summary` page, this simply gives an
-# alternative locate to list the files in the root of the repository.
-#
-# This template executes once per repository.
-files        = "files.html"
-
-# Template responsible for displaying a single file.
-#
-# Called once per parsed file, with both the whole repository and the current
-# file available to the template.
-#
-# This template executes many times.
-file         = "file.html"
-
-# Template responsible for displaying a single directory.
-#
-# Called once per parsed directory, with both the whole repository and the
-# current directory available to the template.
-#
-# This template executes many times.
-dir          = "dir.html"
-
-# Template responsible for displaying a site-wide error.
-#
-# Intended for 404 (page not found) errors.  If used, you must configure your
-# webserver to redirect HTTP errors to the generated error page.
-#
-# This template executes one time.
-error        = "404.html"
-
-
-
-###############################################################################
 ##
-## Subsection specifying names of output files/directories.
+## Next, a `templates` table is defined, which contains a variable
+## number of entries.  Each entry must contain the following three
+## variables:
 ##
-## Each entry pairs with one of the entries in `gitsy_templates` above,
-## specifying the directories and filenames for the rendered output, after the
-## template engine has performed its substitutions.
+## - `template` -- the input file to use as a template, relative to
+##                 `template_root`
 ##
-## All outputs are relative to the `path` directory.
+## - `output` -- the output file(s) to write, relative to
+##                 `output_root`.  These filenames can contain
+##                 variables, which are defined below.
 ##
-## There are currently two supported variables for paths:
+## - `kind` -- the type of output being generated.  This decides which
+##             variables are available in the template, and which
+##             variables can be substituted in the output filename.
 ##
-## * "%REPO%" -- replaced with the name of the currently processing repository
-## * "%ID%" -- replaced with the ID of the currently processing object
-## * "%PAGE%" -- replaced with the current page number if output is paginated
 ##
-## All except `path` are optional.  If not specified, sensible defaults will be
-## used.
+## The following output types are available:
+##
+## - `repo_list` -- Template receives all defined repository metadata in the `repos` variable.
+##
+## - `summary` -- Template receives all metadata of the current repo.
+##                This is split across the `name`, `history`,
+##                `branches`, `tags`, `root_files`, `all_files`,
+##                `commits`, `file_ids`, commit_ids`, `metadata`,
+##                `last_ts_utc`, and `last_ts_offset` variables.
+##
+## - `history` -- All current repo metadata.  `history` variable not
+##                affected by `limit_context`.
+##
+## - `commit` -- All current repo metadata.  Current commit object in
+##               `commit` variable.
+##
+## - `branches` -- All current repo metadata.  `branches` variable not
+##                 affected by `limit_context`.
+##
+## - `branch` -- All current repo metadata.  Current branch object in
+##               `branch` variable.
+##
+## - `tags` -- All current repo metadata.  `tags` variable not
+##             affected by `limit_context`.
+##
+## - `tag` -- All current repo metadata.  Current tag object in `tag`
+##            variable.
+##
+## - `files` -- All current repo metadata.
+##
+## - `file` -- All current repo metadata.  Current file object in
+##             `file` variable.
+##
+## - `dir` -- All current repo metadata.  Current directory object in
+##            `dir` variable.
+##
+## - `error` -- All metadata for all repositories.
+##
+##
+## The following variables are permitted in `output` paths:
+##
+## - "%REPO%" -- Replaced with the name of the currently processing
+##               repository.  Available in all except `repos_list` and
+##               `error`.
+##
+## - "%ID%" -- Replaced with the ID of the currently processing
+##             object.  Available in `commit`, `branch`, `tag`,
+##             `file`, and `dir`.
+##
+## - "%PAGE%" -- Replaced with the current page number if output is
+##               paginated.  Available in `history`, `branches`, and
+##               `tags`.
+##
+##
+## All except `output_root` and `template_root` are optional.
+## Template types that are not specified will not be generated, and
+## all template types can be generated as many times as desired.
 ##
 ###############################################################################
 [gitsy_outputs]
-path = "rendered/"
-repo_list     = "index.html"
-summary  = "%REPO%/index.html"
-history       = "%REPO%/history%PAGE%.html"
-commit        = "%REPO%/commit/%ID%.html"
-branches      = "%REPO%/branches%PAGE%.html"
-branch        = "%REPO%/branch/%ID%.html"
-tags          = "%REPO%/tags%PAGE%.html"
-tag           = "%REPO%/tag/%ID%.html"
-files         = "%REPO%/files.html"
-file          = "%REPO%/file/%ID%.html"
-dir           = "%REPO%/dir/%ID%.html"
-error         = "404.html"
+output_root    = "rendered/"
+template_root  = "templates/default_light/"
+
+templates = [
+      { template = "repos.html",    output = "index.html",                 kind = "repo_list" },
+      { template = "summary.html",  output = "%REPO%/index.html",          kind = "summary"   },
+      { template = "history.html",  output = "%REPO%/history%PAGE%.html",  kind = "history"   },
+      { template = "commit.html",   output = "%REPO%/commit/%ID%.html",    kind = "commit"    },
+      { template = "branches.html", output = "%REPO%/branches%PAGE%.html", kind = "branches"  },
+      { template = "branch.html",   output = "%REPO%/branch/%ID%.html",    kind = "branch"    },
+      { template = "tags.html",     output = "%REPO%/tags%PAGE%.html",     kind = "tags"      },
+      { template = "tag.html",      output = "%REPO%/tag/%ID%.html",       kind = "tag"       },
+      { template = "files.html",    output = "%REPO%/files.html",          kind = "files"     },
+      { template = "file.html",     output = "%REPO%/file/%ID%.html",      kind = "file"      },
+      { template = "dir.html",      output = "%REPO%/dir/%ID%.html",       kind = "dir"       },
+      { template = "404.html",      output = "404.html",                   kind = "error"     }
+]
 
 # Output file for syntax highlighting CSS
 #
 # If syntax highlighting is enabled, a single CSS file will be
 # rendered to this path.  It must be included in the file template to
 # render the syntax highlighting correctly.
+#
+# If not specified, a default is used.
 syntax_css    = "%REPO%/file/syntax.css"
 
 # Output directory for files specified in global `asset_files`.
 #
 # Each input file is copied to this directory unmodified.
+#
+# If not specified, a default is used.
 global_assets = "assets/"
 
 # Output directory for files specified in per-repo `asset_files`.
 #
 # Each input file is copied to this directory unmodified.
+#
+# If not specified, a default is used.
 repo_assets   = "%REPO%/assets/"
 
 # Directory to clone remote repositories into.
@@ -487,9 +431,12 @@ repo_assets   = "%REPO%/assets/"
 # runs, all remote refs are fetched rather than recloning.
 #
 # Only non-authenticated HTTPS repositories are currently supported.
+#
+# If not specified, a default is used.
 cloned_repos  = "cloned_repos/"
 
 
+
 ###############################################################################
 ##
 ## Subsection for arbitrary, global user data.

src/main.rs

@@ -41,6 +41,6 @@ use settings::{GitsyCli, GitsySettings};
 fn main() {
     let cli = GitsyCli::new();
     let (settings, repo_descriptions) = GitsySettings::new(&cli);
-    let generator = GitsyGenerator::new(cli, settings, repo_descriptions);
+    let mut generator = GitsyGenerator::new(cli, settings, repo_descriptions);
     generator.generate().expect("Itsy-Gitsy generation failed!");
 }

src/settings.rs

@@ -20,10 +20,11 @@
  * You should have received a copy of the GNU General Public License
  * along with Itsy-Gitsy.  If not, see <http://www.gnu.org/licenses/>.
  */
-use crate::error;
+use crate::{louder, error};
 use crate::git::GitRepo;
 use crate::util::SafePathVar;
 use clap::Parser;
+use git2::Repository;
 use serde::Deserialize;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fs::{create_dir, create_dir_all, read_dir, remove_dir_all, read_to_string};
@@ -111,56 +112,20 @@ impl GitsyCli {
     }
 }
 
-#[derive(Deserialize, Debug)]
-pub struct GitsySettingsTemplates {
-    pub path: PathBuf,
-    pub repo_list: Option<String>,
-    pub summary: Option<String>,
-    pub history: Option<String>,
-    pub commit: Option<String>,
-    pub branches: Option<String>,
-    pub branch: Option<String>,
-    pub tags: Option<String>,
-    pub tag: Option<String>,
-    pub files: Option<String>,
-    pub file: Option<String>,
-    pub dir: Option<String>,
-    pub error: Option<String>,
-}
-
-impl GitsySettingsTemplates {
-    pub fn template_dir(&self) -> String {
-        self.path.clone().canonicalize()
-            .expect(&format!("ERROR: unable to canonicalize template path: {}", self.path.display()))
-            .to_str().expect(&format!("ERROR: unable to parse template path: {}", self.path.display()))
-            .to_string()
-    }
-}
-
 #[derive(Deserialize, Debug)]
 pub struct GitsySettingsOutputs {
-    pub path: PathBuf,
+    pub output_root: PathBuf,
+    pub template_root: PathBuf,
+    pub templates: Option<Vec<GitsySettingsTemplate>>,
     pub cloned_repos: Option<String>,
-    pub repo_list: Option<String>,
-    pub summary: Option<String>,
-    pub history: Option<String>,
-    pub commit: Option<String>,
-    pub branches: Option<String>,
-    pub branch: Option<String>,
-    pub tags: Option<String>,
-    pub tag: Option<String>,
-    pub files: Option<String>,
-    pub file: Option<String>,
-    pub dir: Option<String>,
-    pub error: Option<String>,
     pub syntax_css: Option<String>,
     pub global_assets: Option<String>,
     pub repo_assets: Option<String>,
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Debug, PartialEq)]
 #[allow(non_camel_case_types)]
-pub enum GitsySettingsExtraType {
+pub enum GitsyTemplateType {
     repo_list,
     summary,
     history,
@@ -171,7 +136,8 @@ pub enum GitsySettingsExtraType {
     tag,
     files,
     file,
-    dir
+    dir,
+    error,
 }
 
 pub fn substitute_path_vars<P,S>(path: &P, repo: Option<&GitRepo>, obj: Option<&S>) -> PathBuf
@@ -185,13 +151,13 @@ where P: AsRef<Path>,
 }
 
 #[derive(Deserialize, Debug)]
-pub struct GitsySettingsExtraOutput {
+pub struct GitsySettingsTemplate {
     pub template: String,
     pub output: String,
-    pub kind: GitsySettingsExtraType,
+    pub kind: GitsyTemplateType,
 }
 
-macro_rules! output_path_fn {
+macro_rules! template_fn {
     ($var:ident, $is_dir:expr, $default:expr) => {
         pub fn $var<S: SafePathVar>(&self, repo: Option<&GitRepo>, obj: Option<&S>) -> PathBuf {
             let tmpl_path = PathBuf::from(self.$var.as_deref().unwrap_or($default));
@@ -201,29 +167,70 @@ macro_rules! output_path_fn {
     }
 }
 
+macro_rules! templates_fn {
+    ($var:ident, $is_dir:expr) => {
+        pub fn $var<S: SafePathVar>(&self, repo: Option<&GitRepo>, obj: Option<&S>) -> Vec<(PathBuf, PathBuf)> {
+            match &self.templates {
+                Some(template) => {
+                    template.iter()
+                        .filter(|x| x.kind == GitsyTemplateType::$var)
+                        .map(|x| {
+                            let tmpl_path = PathBuf::from(&x.output);
+                            let new_path = substitute_path_vars(&tmpl_path, repo, obj);
+                            (PathBuf::from(&x.template),
+                             self.canonicalize_and_create(&new_path, $is_dir))
+                        }).collect()
+                },
+                None => {
+                    vec!()
+                },
+            }
+        }
+    }
+}
+
+impl SafePathVar for GitsySettingsOutputs {
+    fn safe_substitute(&self, path: &impl AsRef<Path>) -> PathBuf {
+        let src: &Path = path.as_ref();
+        let mut dst = PathBuf::new();
+        let root = self.template_root.to_str()
+            .expect(&format!("ERROR: couldn't parse template root: {}", self.template_root.display()));
+        for cmp in src.components() {
+            // NOTE: this variable is not sanitized, since it's
+            // allowed to create new directory structure.
+            let cmp = cmp.as_os_str().to_string_lossy()
+                .replace("%TEMPLATE%", &root);
+            dst.push(cmp);
+        }
+        dst
+    }
+}
+
 #[rustfmt::skip]
 impl GitsySettingsOutputs {
-    output_path_fn!(repo_list,     false, "index.html");
-    output_path_fn!(summary,       false, "%REPO%/index.html");
-    output_path_fn!(history,       false, "%REPO%/history%PAGE%.html");
-    output_path_fn!(commit,        false, "%REPO%/commit/%ID%.html");
-    output_path_fn!(branches,      false, "%REPO%/branches%PAGE%.html");
-    output_path_fn!(branch,        false, "%REPO%/branch/%ID%.html");
-    output_path_fn!(tags,          false, "%REPO%/tags%PAGE%.html");
-    output_path_fn!(tag,           false, "%REPO%/tag/%ID%.html");
-    output_path_fn!(files,         false, "%REPO%/files.html");
-    output_path_fn!(file,          false, "%REPO%/file/%ID%.html");
-    output_path_fn!(syntax_css,    false, "%REPO%/file/syntax.css");
-    output_path_fn!(dir,           false, "%REPO%/dir/%ID%.html");
-    output_path_fn!(error,         false, "404.html");
-    output_path_fn!(global_assets, true,  "assets/");
-    output_path_fn!(repo_assets,   true,  "%REPO%/assets/");
+    // Single entries:
+    template_fn!(syntax_css,    false, "%REPO%/file/syntax.css");
+    template_fn!(global_assets, true,  "assets/");
+    template_fn!(repo_assets,   true,  "%REPO%/assets/");
+    // Zero or more entries (Vec):
+    templates_fn!(repo_list, false);
+    templates_fn!(summary, false);
+    templates_fn!(history, false);
+    templates_fn!(commit, false);
+    templates_fn!(branches, false);
+    templates_fn!(branch, false);
+    templates_fn!(tags, false);
+    templates_fn!(tag, false);
+    templates_fn!(files, false);
+    templates_fn!(file, false);
+    templates_fn!(dir, false);
+    templates_fn!(error, false);
 
     fn canonicalize_and_create(&self, path: &Path, is_dir: bool) -> PathBuf {
-        let mut canonical_path = self.path.clone()
+        let mut canonical_path = self.output_root.clone()
             .canonicalize().expect(&format!(
                 "ERROR: unable to canonicalize output path: {}",
-                self.path.display()));
+                self.output_root.display()));
         canonical_path.push(path);
         match is_dir {
             true => {
@@ -238,21 +245,49 @@ impl GitsySettingsOutputs {
         canonical_path
     }
 
-    pub fn output_dir(&self) -> String {
-        self.path.clone().canonicalize()
-            .expect(&format!("ERROR: unable to canonicalize output path: {}", self.path.display()))
-            .to_str().expect(&format!("ERROR: unable to parse output path: {}", self.path.display()))
-            .to_string()
+    pub fn output_dir(&self) -> PathBuf {
+        self.output_root.clone().canonicalize()
+            .expect(&format!("ERROR: unable to canonicalize output path: {}", self.output_root.display()))
+    }
+
+    pub fn template_dir(&self) -> PathBuf {
+        self.template_root.clone().canonicalize()
+            .expect(&format!("ERROR: unable to canonicalize template path: {}", self.template_root.display()))
+    }
+
+    pub fn has_files(&self) -> bool {
+        match &self.templates {
+            Some(template) => template.iter().filter(|x| x.kind == GitsyTemplateType::file).count() > 0,
+            _ => false,
+        }
+    }
+
+    pub fn asset<P: AsRef<Path>>(&self, asset: &P, parsed_repo: Option<&GitRepo>, repo: Option<&Repository>) -> PathBuf {
+        let tmpl_path = asset.as_ref().to_path_buf();
+        let asset_path = substitute_path_vars(&tmpl_path, parsed_repo, Some(self));
+        let full_path = match repo {
+            Some(repo) => {
+                let mut full_path = repo.path().to_owned();
+                full_path.push(asset_path);
+                full_path
+            },
+            _ => {
+                asset_path
+            }
+        };
+        full_path
     }
 
     pub fn create(&self) {
-        let _ = create_dir(self.path.to_str().expect(&format!("ERROR: output path invalid: {}", self.path.display())));
+        louder!("Creating output directory: {}", self.output_root.display());
+        let _ = create_dir(self.output_root.to_str().expect(&format!("ERROR: output path invalid: {}", self.output_root.display())));
     }
 
     pub fn clean(&self) {
-        if !self.path.exists() {
+        if !self.output_root.exists() {
             return;
         }
+        louder!("Cleaning output directory: {}", self.output_root.display());
         let dir: PathBuf = PathBuf::from(&self.output_dir());
         assert!(dir.is_dir(), "ERROR: Output directory is... not a directory? {}", dir.display());
         remove_dir_all(&dir)
@@ -270,6 +305,41 @@ impl GitsySettingsOutputs {
             .expect(&format!("ERROR: Unable to make path relative: {}", path))
             .to_string()
     }
+
+    pub fn assert_valid<P: AsRef<Path>>(&self, path: &P) -> bool {
+        let path = path.as_ref().to_str()
+            .expect(&format!("ERROR: attempted to write unrecognizeable path: {}", path.as_ref().display()));
+        // Ensure that the requested output path is actually a child
+        // of the output directory, as a sanity check to ensure we
+        // aren't writing out of bounds.
+        let canonical_root = self.output_root.canonicalize().expect(&format!(
+            "Cannot find canonical version of output path: {}",
+            self.output_root.display()
+        ));
+        let canonical_path = PathBuf::from(path);
+        let has_relative_dirs = canonical_path
+            .ancestors()
+            .any(|x| x.file_name().is_none() && x != Path::new("/"));
+        assert!(
+            canonical_path.is_absolute(),
+            "ERROR: write_rendered called with a relative path: {}",
+            path
+        );
+        assert!(
+            !has_relative_dirs,
+            "ERROR: write_rendered called with a relative path: {}",
+            path
+        );
+        let _ = canonical_path
+            .ancestors()
+            .find(|x| x == &canonical_root)
+            .expect(&format!(
+                "Output file {} not contained in output path: {}",
+                canonical_path.display(),
+                canonical_root.display()
+            ));
+        true
+    }
 }
 
 #[derive(Clone, Deserialize, Default, Debug)]
@@ -326,10 +396,6 @@ pub struct GitsySettings {
     pub readme_files: Option<Vec<String>>,
     pub asset_files: Option<Vec<String>>,
     pub branch: Option<String>,
-    #[serde(rename(deserialize = "gitsy_templates"))]
-    pub templates: GitsySettingsTemplates,
-    #[serde(rename(deserialize = "gitsy_outputs"))]
-    pub outputs: GitsySettingsOutputs,
     pub paginate_history: Option<usize>,
     pub paginate_branches: Option<usize>,
     pub paginate_tags: Option<usize>,
@@ -346,6 +412,8 @@ pub struct GitsySettings {
     pub render_markdown: Option<bool>,
     pub syntax_highlight: Option<bool>,
     pub syntax_highlight_theme: Option<String>,
+    #[serde(rename(deserialize = "gitsy_outputs"))]
+    pub outputs: GitsySettingsOutputs,
     #[serde(rename(deserialize = "gitsy_extra"))]
     pub extra: Option<BTreeMap<String, toml::Value>>,
 }