add support for cloning remote repos (non-authed HTTPS only)

Cargo.toml

@@ -26,3 +26,7 @@ serde = { version = "1.0.152", features = ["derive"] }
 syntect = { version = "5.0.0", default-features = false, optional = true }
 tera = "1.17.1"
 toml = "0.5.10"
+
+# For SSH passphrase support:
+#pinentry = "0.5.0"
+#secrecy = "0.8.0"

settings.toml

@@ -300,6 +300,15 @@ global_assets = "assets/"
 # Each input file is copied to this directory unmodified.
 repo_assets   = "%REPO%/assets/"
 
+# Directory to clone remote repositories into.
+#
+# Remote repositories must be cloned locally to parse.  They are cloned into
+# subdirectories of `cloned_repos`, as bare git repos.  These are not deleted
+# when Itsy-Gitsy finishes.  If the directories already exist when Itsy-Gitsy
+# runs, all remote refs are fetched rather than recloning.
+#
+# Only non-authenticated HTTPS repositories are currently supported.
+cloned_repos  = "bare_repos/"
 
 
 ###############################################################################
@@ -370,3 +379,6 @@ attributes = { status = "active", type = "daemon" }
 #[circadian.attributes]
 #status = "active"
 #type = "daemon"
+
+[ossuary]
+path = "https://github.com/mrmekon/ossuary"

src/main.rs

@@ -777,6 +777,7 @@ struct GitsySettingsTemplates {
 #[derive(Deserialize, Debug)]
 struct GitsySettingsOutputs {
     path: PathBuf,
+    cloned_repos: Option<String>,
     repo_list: Option<String>,
     repo_summary: Option<String>,
     commit: Option<String>,
@@ -1013,14 +1014,84 @@ fn main() {
     // Iterate over each repository, generating outputs
     for repo_desc in &repo_vec {
         loudest!("Repo settings:\n{:#?}", &repo_desc);
+        let start_repo = std::time::Instant::now();
         let mut repo_bytes = 0;
-        let dir = &repo_desc.path;
+        let name = repo_desc.name.as_deref().expect("A configured repository has no name!");
+
+        let repo_path = match &repo_desc.path {
+            url if url.starts_with("https://") ||
+                url.to_str().unwrap_or_default().contains("@") => {
+                if settings.outputs.cloned_repos.is_none() {
+                    error!("ERROR: Found remote repo [{}], but `cloned_repos` directory not configured.", name);
+                    continue;
+                };
+                let clone_path: PathBuf = [settings.outputs.cloned_repos.as_deref().unwrap(),
+                                           name].iter().collect();
+                match Repository::open(&clone_path) {
+                    Ok(r) => {
+                        // Repo already cloned, so update all refs
+                        let refs: Vec<String> = r.references()
+                            .expect(&format!("Unable to enumerate references for repo [{}]", name))
+                            .map(|x| x.expect(&format!("Found invalid reference in repo [{}]", name))
+                                 .name()
+                                 .expect(&format!("Found unnamed reference in repo: [{}]", name))
+                                 .to_string()).collect();
+                        r.find_remote("origin")
+                            .expect(&format!("Clone of repo [{}] missing `origin` remote.", name))
+                            .fetch(&refs, None, None)
+                            .expect(&format!("Failed to fetch updates from remote repo [{}]", name));
+                        clone_path.to_string_lossy().to_string()
+                    },
+                    Err(_) => {
+                        let mut builder = git2::build::RepoBuilder::new();
+
+                        // TODO: git2-rs's ssh support just doesn't seem to
+                        // work.  It finds the repo, but fails to either
+                        // decrypt or use the private key.
+                        //
+                        //if !url.starts_with("https://") {
+                        //    use secrecy::ExposeSecret;
+                        //    // this must be SSH, which needs credentials.
+                        //    let mut callbacks = git2::RemoteCallbacks::new();
+                        //    callbacks.credentials(|_url, username_from_url, _allowed_types| {
+                        //        //git2::Cred::ssh_key_from_agent(username_from_url.unwrap())
+                        //
+                        //        let keyfile = format!("{}/.ssh/id_rsa", std::env::var("HOME").unwrap());
+                        //        let passphrase = pinentry::PassphraseInput::with_default_binary().unwrap()
+                        //            .with_description(&format!("Enter passphrase for SSH key {} (repo: {})",
+                        //                                       keyfile, url.display()))
+                        //            .with_prompt("Passphrase:")
+                        //            .interact().unwrap();
+                        //        git2::Cred::ssh_key(
+                        //            username_from_url.unwrap(),
+                        //            None,
+                        //            Path::new(&keyfile),
+                        //            Some(passphrase.expose_secret()),
+                        //        )
+                        //    });
+                        //    let mut options = git2::FetchOptions::new();
+                        //    options.remote_callbacks(callbacks);
+                        //    builder.fetch_options(options);
+                        //}
+                        builder
+                            .bare(true)
+                            .clone(&url.to_string_lossy().to_string(), &clone_path)
+                            .expect(&format!("Failed to clone remote repo [{}]", name));
+                        clone_path.to_string_lossy().to_string()
+                    }
+                }
+            }
+            dir => {
                 match dir.metadata() {
                     Ok(m) if m.is_dir() => {},
-            _ => continue,
+                    _ => {
+                        error!("ERROR: local repository [{}]: directory not found: {}", name, dir.display());
+                        continue;
+                    },
                 }
-        let repo_path: String = dir.to_string_lossy().to_string();
+                dir.to_string_lossy().to_string()
-        let name = repo_desc.name.as_deref().unwrap_or_default();
+            },
+        };
         let repo = Repository::open(&repo_path).expect("Unable to find git repository.");
         let metadata = GitsyMetadata {
             full_name: repo_desc.name.clone(),
@@ -1030,7 +1101,6 @@ fn main() {
             attributes: repo_desc.attributes.clone().unwrap_or_default(),
         };
         normal_noln!("[{}{}]... ", name, " ".repeat(longest_repo_name - name.len()));
-        let start_parse = std::time::Instant::now();
         let summary = parse_repo(&repo, &name, &repo_desc, metadata).expect("Failed to analyze repo HEAD.");
 
         let mut local_ctx = Context::from_serialize(&summary).unwrap();
@@ -1184,7 +1254,7 @@ fn main() {
                     true => " - ",
                     _ => "",
                 },
-                start_parse.elapsed().as_secs_f32(), repo_bytes);
+                start_repo.elapsed().as_secs_f32(), repo_bytes);
         total_bytes += repo_bytes;
         size_check!(repo_desc, repo_bytes, total_bytes);
     }