/* * Copyright 2023 Trevor Bentley * * Author: Trevor Bentley * Contact: gitsy@@trevorbentley.com * Source: https://github.com/mrmekon/itsy-gitsy * * This file is part of Itsy-Gitsy. * * Itsy-Gitsy is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * Itsy-Gitsy is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with Itsy-Gitsy. If not, see . */ use crate::settings::GitsySettingsRepo; use crate::util::{sanitize_path_component, SafePathVar, urlify_path}; use crate::{error, loud, loudest}; use git2::{DiffOptions, Error, Repository}; use serde::{Deserialize, Serialize}; use std::cell::RefCell; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::rc::Rc; use std::sync::atomic::Ordering; fn first_line(msg: &[u8]) -> String { let message = String::from_utf8_lossy(msg); message.lines().next().unwrap_or("[no commit message]").to_owned() } #[derive(Serialize, Default)] pub struct GitRepo { pub name: String, pub last_ts_utc: i64, pub last_ts_offset: i64, pub metadata: GitsyMetadata, pub history: Vec, pub branches: Vec, pub tags: Vec, pub root_files: Vec, pub all_files: Vec, pub commits: BTreeMap, // TODO: this is duplication that should be handled with // references. Used so templates can deduce which files have been // generated. pub commit_ids: Vec, pub file_ids: Vec, } impl GitRepo { pub fn minimal_clone(&self, max_entries: usize) -> Self { let mut new_commits: BTreeMap = BTreeMap::new(); let new_history: Vec = self.history.iter().cloned().take(max_entries).collect(); for entry in &new_history { if self.commits.contains_key(&entry.full_hash) { new_commits.insert(entry.full_hash.clone(), self.commits.get(&entry.full_hash).unwrap().clone()); } } let all_files: Vec = self.all_files.iter().cloned().take(max_entries).collect(); GitRepo { name: self.name.clone(), last_ts_utc: self.last_ts_utc, last_ts_offset: self.last_ts_offset, metadata: self.metadata.clone(), history: new_history, branches: self.branches.iter().cloned().take(max_entries).collect(), tags: self.tags.iter().cloned().take(max_entries).collect(), // Don't minimize the root tree, because that's weird UX // for the summary page. root_files: self.root_files.clone(), all_files, commits: new_commits, // These are not minimized because they're a listing of // which generated files should exist, and are needed for // ensuring valid links on every page. file_ids: self.file_ids.clone(), commit_ids: self.commit_ids.clone(), } } } impl SafePathVar for GitRepo { fn safe_substitute(&self, path: &impl AsRef) -> PathBuf { let src: &Path = path.as_ref(); let mut dst = PathBuf::new(); let safe_name = sanitize_path_component(&self.name); for cmp in src.components() { let cmp = cmp.as_os_str().to_string_lossy().replace("%REPO%", &safe_name); dst.push(cmp); } assert!(src.components().count() == dst.components().count(), "ERROR: path substitution accidentally created a new folder in: {}", src.display()); dst } } #[derive(Clone, Serialize, Default)] pub struct GitsyMetadata { pub full_name: Option, pub description: Option, pub website: Option, pub clone: Option, pub attributes: BTreeMap, } #[derive(Clone, Serialize, Default)] pub struct GitAuthor { pub name: Option, pub email: Option, } #[derive(Clone, Serialize, Default)] pub struct GitObject { pub full_hash: String, pub short_hash: String, pub ts_utc: i64, pub ts_offset: i64, pub author: GitAuthor, pub committer: GitAuthor, pub parents: Vec, pub ref_name: Option, pub alt_refs: Vec, pub tagged_id: Option, pub tree_id: Option, pub summary: Option, pub message: Option, pub stats: Option, pub diff: Option, } impl SafePathVar for GitObject { fn safe_substitute(&self, path: &impl AsRef) -> PathBuf { let src: &Path = path.as_ref(); let mut dst = PathBuf::new(); let safe_full_hash = sanitize_path_component(&self.full_hash); let safe_ref = self.ref_name.as_deref() .map(|v| sanitize_path_component(&v)) .unwrap_or("%REF%".to_string()); for cmp in src.components() { let cmp = cmp.as_os_str().to_string_lossy() .replace("%ID%", &safe_full_hash) .replace("%REF%", &safe_ref); dst.push(cmp); } assert!(src.components().count() == dst.components().count(), "ERROR: path substitution accidentally created a new folder in: {}", src.display()); dst } } #[derive(Clone, Serialize, Default)] pub struct GitStats { pub files: usize, pub additions: usize, pub deletions: usize, } #[derive(Serialize, Deserialize, Clone)] pub struct GitFile { pub id: String, pub name: String, pub path: String, pub mode: i32, pub kind: String, pub is_binary: bool, pub size: usize, pub tree_depth: usize, pub contents: Option, pub contents_safe: bool, pub contents_preformatted: bool, } impl SafePathVar for GitFile { fn safe_substitute(&self, path: &impl AsRef) -> PathBuf { let src: &Path = path.as_ref(); let mut dst = PathBuf::new(); let safe_id = sanitize_path_component(&self.id); let safe_name = sanitize_path_component(&self.name); let safe_path = sanitize_path_component(&urlify_path(&self.path)); for cmp in src.components() { let cmp = cmp.as_os_str().to_string_lossy() .replace("%ID%", &safe_id) .replace("%NAME%", &safe_name) .replace("%PATH%", &safe_path); dst.push(cmp); } assert!(src.components().count() == dst.components().count(), "ERROR: path substitution accidentally created a new folder in: {}", src.display()); dst } } #[derive(Clone, Serialize, Default)] pub struct GitDiffCommit { pub files: Vec, pub file_count: usize, pub additions: usize, pub deletions: usize, } #[derive(Clone, Serialize, Default)] pub struct GitDiffFile { pub oldfile: String, pub newfile: String, pub basefile: String, pub oldid: String, pub newid: String, pub extra: String, pub additions: usize, pub deletions: usize, pub hunks: Vec, } #[derive(Clone, Serialize, Default)] pub struct GitDiffHunk { pub context: String, pub lines: Vec, } #[derive(Clone, Serialize)] pub struct GitDiffLine { pub kind: &'static str, pub prefix: &'static str, pub text: String, } fn walk_file_tree( repo: &git2::Repository, rev: &str, files: &mut Vec, depth: usize, max_depth: usize, recurse: bool, prefix: &str, ) -> Result<(), Error> { let obj = repo.revparse_single(rev)?; let tree = obj.peel_to_tree()?; for entry in tree.iter() { let name = entry.name().unwrap_or_default().to_string(); let path = prefix.to_string() + entry.name().unwrap_or_default(); let kind = match entry.kind() { Some(git2::ObjectType::Tree) => "dir", Some(git2::ObjectType::Blob) => "file", Some(git2::ObjectType::Commit) => "submodule", _ => "unknown", }; let mut is_binary = false; let mut size = 0; if let Ok(blob) = repo.find_blob(entry.id()) { is_binary = blob.is_binary(); size = blob.content().len(); } loudest!(" + file: {}", path); files.push(GitFile { id: entry.id().to_string(), name: name.clone(), path: path.clone(), kind: kind.to_string(), mode: entry.filemode(), is_binary, size, tree_depth: depth, contents: None, contents_safe: false, contents_preformatted: true, }); if recurse && depth < (max_depth - 1) && entry.kind() == Some(git2::ObjectType::Tree) { let prefix = path + "/"; walk_file_tree( repo, &entry.id().to_string(), files, depth + 1, max_depth, true, &prefix, )?; } } Ok(()) } pub fn dir_listing(repo: &Repository, file: &GitFile) -> Result, Error> { let mut files: Vec = vec![]; walk_file_tree(&repo, &file.id, &mut files, 0, usize::MAX, false, "")?; Ok(files) } pub fn parse_repo( repo: &Repository, name: &str, settings: &GitsySettingsRepo, metadata: GitsyMetadata, ) -> Result { let mut history: Vec = vec![]; let mut branches: Vec = vec![]; let mut tags: Vec = vec![]; let mut commits: BTreeMap = BTreeMap::new(); let mut commit_count = 0; let mut history_count = 0; let mut branch_count = 0; let mut tag_count = 0; let branch_name = settings.branch.as_deref().unwrap_or("master"); let branch_obj = repo.revparse_single(branch_name)?; loud!(); // Cache the shortnames of all references loudest!(" - Parsing references"); let mut references: BTreeMap> = BTreeMap::new(); for refr in repo.references()? { let refr = refr?; if let (Some(target), Some(name)) = (refr.target(), refr.shorthand()) { let id = match refr.peel_to_tag() { Ok(tag) => tag.target_id().to_string(), _ => target.to_string(), }; match references.contains_key(&id) { false => { references.insert(id, vec![name.to_string()]); } true => { references.get_mut(&id).unwrap().push(name.to_string()); } } } } loud!(" - parsed {} references", references.len()); let mut revwalk = repo.revwalk()?; // TODO: TOPOLOGICAL might be better, but it's also ungodly slow // on large repos. Maybe this should be configurable. // //revwalk.set_sorting(git2::Sort::TOPOLOGICAL)?; revwalk.set_sorting(git2::Sort::NONE)?; revwalk.push(branch_obj.id())?; loudest!(" - Parsing history:"); for (idx, oid) in revwalk.by_ref().enumerate() { let oid = oid?; if commit_count >= settings.limit_commits.unwrap_or(usize::MAX) || history_count >= settings.limit_history.unwrap_or(usize::MAX) { break; } commits.insert(oid.to_string(), parse_commit(idx, settings, repo, &oid.to_string())?); commit_count += 1; let commit = repo.find_commit(oid)?; let obj = repo.revparse_single(&commit.id().to_string())?; let full_hash = commit.id().to_string(); let short_hash = obj.short_id()?.as_str().unwrap_or_default().to_string(); let mut parents: Vec = vec![]; let a = match commit.parents().len() { x if x == 1 => { let parent = commit.parent(0).unwrap(); parents.push(parent.id().to_string()); Some(parent.tree()?) } x if x > 1 => { for parent in commit.parents() { parents.push(parent.id().to_string()); } let parent = commit.parent(0).unwrap(); Some(parent.tree()?) } _ => None, }; let b = commit.tree()?; let mut diffopts = DiffOptions::new(); let stats = match idx < settings.limit_diffs.unwrap_or(usize::MAX) { true => { let diff = repo.diff_tree_to_tree(a.as_ref(), Some(&b), Some(&mut diffopts))?; let stats = diff.stats()?; let stats = GitStats { files: stats.files_changed(), additions: stats.insertions(), deletions: stats.deletions(), }; Some(stats) }, false => { None }, }; let alt_refs: Vec = references .get(&commit.id().to_string()) .map(|x| x.to_owned()) .unwrap_or_default(); if history_count < settings.limit_history.unwrap_or(usize::MAX) { loudest!(" + {} {}", full_hash, first_line(commit.message_bytes())); // TODO: this is basically a duplicate of the commit // array, and really should be pointers to that array // instead. But it's not a quick task to switch to // self-referential data structures in Rust. history.push(GitObject { full_hash, short_hash, ts_utc: commit.author().when().seconds(), ts_offset: (commit.author().when().offset_minutes() as i64) * 60, parents, ref_name: None, alt_refs, author: GitAuthor { name: commit.author().name().map(|x| x.to_owned()), email: commit.author().email().map(|x| x.to_owned()), }, summary: Some(first_line(commit.message_bytes())), stats, ..Default::default() }); history_count += 1; } } loud!(" - parsed {} history entries", history_count); loud!(" - parsed {} commits", commit_count); loudest!(" - Parsing branches:"); for branch in repo.branches(None)? { if branch_count >= settings.limit_branches.unwrap_or(usize::MAX) { break; } let (branch, _branch_type) = branch?; let refr = branch.get(); let name = branch.name()?.unwrap_or("[unnamed]"); let obj = repo.revparse_single(name)?; // Only show direct references, skip symbolic aliases. Maybe // this is a bad idea? match refr.kind() { Some(k) if k == git2::ReferenceType::Symbolic => continue, _ => {} } let commit = repo.find_commit(obj.id())?; let full_hash = obj.id().to_string(); let short_hash = obj.short_id()?.as_str().unwrap_or_default().to_string(); loudest!(" + {} {}", full_hash, name); branches.push(GitObject { full_hash, short_hash, ts_utc: commit.author().when().seconds(), ts_offset: (commit.author().when().offset_minutes() as i64) * 60, parents: vec![], ref_name: Some(name.to_string()), author: GitAuthor { name: commit.author().name().map(|x| x.to_owned()), email: commit.author().email().map(|x| x.to_owned()), }, committer: GitAuthor { name: commit.committer().name().map(|x| x.to_owned()), email: commit.committer().email().map(|x| x.to_owned()), }, summary: Some(first_line(commit.message_bytes())), message: commit.message().map(|x| x.to_string()), ..Default::default() }); branch_count += 1; } loud!(" - parsed {} branches", branch_count); loudest!(" - Parsing tags:"); for tag in repo.tag_names(None)?.iter().rev() { if tag_count >= settings.limit_tags.unwrap_or(usize::MAX) { break; } let tag = tag.unwrap_or("[unnamed]"); let obj = repo.revparse_single(tag)?; let full_hash = obj.id().to_string(); let short_hash = obj.short_id()?.as_str().unwrap_or_default().to_string(); let commit = match repo.find_tag(obj.id()) { Ok(c) => c, Err(_e) => { error!("WARNING: tag commit not found for tag: {}", obj.id().to_string()); tags.push(GitObject { full_hash, short_hash, ref_name: Some(tag.to_string()), ..Default::default() }); tag_count += 1; continue; } }; let (ts, tz) = match commit.tagger() { Some(t) => (t.when().seconds(), (t.when().offset_minutes() as i64) * 60), _ => (0, 0), }; let (author, email) = match commit.tagger() { Some(t) => (t.name().map(|x| x.to_owned()), t.email().map(|x| x.to_owned())), _ => (None, None), }; let summary = match commit.message_bytes() { Some(m) => Some(first_line(m)), _ => None, }; loudest!(" + {} {}", full_hash, tag); tags.push(GitObject { full_hash, short_hash, ts_utc: ts, ts_offset: tz, ref_name: Some(tag.to_string()), author: GitAuthor { name: author, email }, tagged_id: Some(commit.target_id().to_string()), message: commit.message().map(|x| x.to_string()), summary, ..Default::default() }); tag_count += 1; } loud!(" - parsed {} tags", tag_count); let mut root_files: Vec = vec![]; let mut all_files: Vec = vec![]; let max_depth = settings.limit_tree_depth.unwrap_or(usize::MAX); if max_depth > 0 { loudest!(" - Walking root files"); walk_file_tree(&repo, branch_name, &mut root_files, 0, usize::MAX, false, "")?; // TODO: maybe this should be optional? Walking the whole tree // could be slow on huge repos. loudest!(" - Walking all files"); walk_file_tree(&repo, branch_name, &mut all_files, 0, max_depth, true, "")?; } loud!(" - parsed {} files", all_files.len()); let file_ids = all_files.iter().map(|x| x.id.clone()).collect(); let commit_ids = commits.keys().cloned().collect(); Ok(GitRepo { name: name.to_string(), last_ts_utc: history.first().map(|x| x.ts_utc).unwrap_or(0), last_ts_offset: history.first().map(|x| x.ts_offset).unwrap_or(0), metadata, history, branches, tags, root_files, all_files, commits, commit_ids, file_ids, }) } pub fn parse_commit(idx: usize, settings: &GitsySettingsRepo, repo: &Repository, refr: &str) -> Result { let obj = repo.revparse_single(refr)?; let commit = repo.find_commit(obj.id())?; let mut parents: Vec = vec![]; let a = match commit.parents().len() { x if x == 1 => { let parent = commit.parent(0).unwrap(); parents.push(parent.id().to_string()); Some(parent.tree()?) } x if x > 1 => { for parent in commit.parents() { parents.push(parent.id().to_string()); } let parent = commit.parent(0).unwrap(); Some(parent.tree()?) } _ => None, }; let b = commit.tree()?; let mut diffopts = DiffOptions::new(); diffopts.enable_fast_untracked_dirs(true); let diff = repo.diff_tree_to_tree(a.as_ref(), Some(&b), Some(&mut diffopts))?; let commit_diff: Option = match idx < settings.limit_diffs.unwrap_or(usize::MAX) { true => { let stats = diff.stats()?; Some(GitDiffCommit { file_count: stats.files_changed(), additions: stats.insertions(), deletions: stats.deletions(), ..Default::default() }) }, false => { None } }; let commit_diff = match commit_diff { None => None, Some(mut commit_diff) => { let files: Rc>> = Rc::new(RefCell::new(vec![])); diff.foreach( &mut |file, _progress| { let mut file_diff: GitDiffFile = Default::default(); file_diff.newfile = match file.status() { git2::Delta::Deleted => "/dev/null".to_owned(), _ => file .new_file() .path() .map(|x| "b/".to_string() + &x.to_string_lossy()) .unwrap_or("/dev/null".to_string()), }; file_diff.oldfile = match file.status() { git2::Delta::Added => "/dev/null".to_owned(), _ => file .old_file() .path() .map(|x| "a/".to_string() + &x.to_string_lossy()) .unwrap_or("/dev/null".to_string()), }; file_diff.basefile = match file.status() { git2::Delta::Added => file .new_file() .path() .map(|x| x.to_string_lossy().to_string()) .unwrap_or("/dev/null".to_string()), _ => file .old_file() .path() .map(|x| x.to_string_lossy().to_string()) .unwrap_or("/dev/null".to_string()), }; file_diff.oldid = file.old_file().id().to_string(); file_diff.newid = file.new_file().id().to_string(); files.borrow_mut().push(file_diff); true }, None, // TODO: handle binary files? Some(&mut |_file, hunk| { let mut files = files.borrow_mut(); let file_diff: &mut GitDiffFile = files.last_mut().expect("Diff hunk not associated with a file!"); let mut hunk_diff: GitDiffHunk = Default::default(); hunk_diff.context = String::from_utf8_lossy(hunk.header()).to_string(); file_diff.hunks.push(hunk_diff); true }), Some(&mut |_file, _hunk, line| { let mut files = files.borrow_mut(); let file_diff: &mut GitDiffFile = files.last_mut().expect("Diff hunk not associated with a file!"); let hunk_diff: &mut GitDiffHunk = file_diff .hunks .last_mut() .expect("Diff line not associated with a hunk!"); let (kind, prefix) = match line.origin() { ' ' => ("ctx", " "), '-' => ("del", "-"), '+' => ("add", "+"), _ => ("other", " "), }; match line.origin() { '-' => file_diff.deletions += 1, '+' => file_diff.additions += 1, _ => {} } let line_diff = GitDiffLine { text: String::from_utf8_lossy(line.content()).to_string(), kind, prefix, }; hunk_diff.lines.push(line_diff); true }), )?; match Rc::try_unwrap(files) { Ok(files) => { let files: Vec = files.into_inner(); commit_diff.files = files; } Err(_) => {} } Some(commit_diff) } }; let tree = obj.peel_to_tree()?; let summary = GitObject { full_hash: obj.id().to_string(), short_hash: obj.short_id()?.as_str().unwrap_or_default().to_string(), ts_utc: commit.author().when().seconds(), ts_offset: (commit.author().when().offset_minutes() as i64) * 60, tagged_id: None, tree_id: Some(tree.id().to_string()), parents, ref_name: None, alt_refs: vec![], author: GitAuthor { name: commit.author().name().map(|x| x.to_string()), email: commit.author().email().map(|x| x.to_string()), }, committer: GitAuthor { name: commit.committer().name().map(|x| x.to_string()), email: commit.committer().email().map(|x| x.to_string()), }, summary: Some(first_line(commit.message_bytes())), message: commit.message().map(|x| x.to_string()), stats: None, diff: commit_diff, }; Ok(summary) }