//! The git bridge: git history in, and git history out to a host. //! //! vcs reads git repositories with the git on this machine (`git cat-file //! --batch`). Each commit on the default branch's first-parent line becomes //! a version; each git file and folder becomes a vcs object once, however //! many commits share it. A git branch can come in as a change. Publishing //! an imported project copies its original git history (branches and tags, //! byte for byte) to the host's git view, which then carries on from it. use std::collections::HashMap; use std::io::{BufRead, BufReader, IsTerminal, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; use std::time::Instant; use anyhow::{Result, anyhow, bail}; use base64::Engine; use serde_json::json; use vcs_core::object::{Kind, Object}; use vcs_core::tree::{self, EntryKind, TreeEntry}; use vcs_core::version::Version; use vcs_core::Id; use crate::app::App; use crate::db::{Change, Project, Save, Workspace}; use crate::ids; use crate::objects::Store; use crate::out::{Fail, Out, Report}; use crate::ws::{self, Index}; /// A git repository to read: one on this machine, or a temporary mirror of /// a remote one (removed when dropped). pub struct Repo { pub dir: PathBuf, temp: Option, } impl Repo { /// Bytes in its object IDs: 32 for a SHA-256 repository, 20 for SHA-1. pub fn id_width(&self) -> usize { match git(&self.dir, &["rev-parse", "--show-object-format"]) { Ok(f) if f.trim() == "sha256" => 32, _ => 20, } } } impl Drop for Repo { fn drop(&mut self) { if let Some(t) = &self.temp { let _ = std::fs::remove_dir_all(t); } } } fn git_missing() -> Fail { Fail::new("failed", "reading git repositories needs git installed").state("nothing changed") } /// Runs git in `dir` and returns what it printed. fn git(dir: &Path, args: &[&str]) -> Result { let out = Command::new("git").arg("--git-dir").arg(dir).args(args).stdin(Stdio::null()).output().map_err(|_| git_missing())?; if !out.status.success() { let err = String::from_utf8_lossy(&out.stderr).trim().to_string(); return Err(Fail::new("failed", format!("git {}: {err}", args.first().unwrap_or(&""))).state("nothing changed")); } Ok(String::from_utf8_lossy(&out.stdout).into_owned()) } /// The clone URL for a project address or URL (Upstream projects clone from git.coline.app). fn clone_url(text: &str) -> String { let t = text.trim().trim_end_matches('/'); let bare = t.strip_prefix("https://").or_else(|| t.strip_prefix("http://")).unwrap_or(t); if let Some(rest) = bare.strip_prefix("upstream.codes/") { let rest = rest.strip_suffix(".git").unwrap_or(rest); if rest.split('/').count() == 2 { return format!("https://git.coline.app/{rest}.git"); } } if t.contains("://") || t.contains('@') { t.to_string() } else { format!("https://{t}") } } /// Opens a git repository: a folder on this machine, or a URL (mirrored to a temporary folder). pub fn open(app: &App, source: &str) -> Result { open_as(app, source, None) } /// Like open, cloning with `token` as the password (through git's config /// variables in the environment, so it isn't on the command line). pub fn open_as(app: &App, source: &str, token: Option<&str>) -> Result { let local = app.cwd.join(source); if local.exists() { let out = Command::new("git").arg("-C").arg(&local).args(["rev-parse", "--absolute-git-dir"]).output().map_err(|_| git_missing())?; if !out.status.success() { return Err(Fail::new("bad_usage", format!("{source} isn't a git repository")).state("nothing changed")); } return Ok(Repo { dir: PathBuf::from(String::from_utf8_lossy(&out.stdout).trim()), temp: None }); } let url = clone_url(source); let temp = crate::paths::home().join("tmp").join(format!("import-{}", ids::new_workspace_id())); std::fs::create_dir_all(temp.parent().unwrap())?; let mut clone = Command::new("git"); if let Some(t) = token { let basic = base64::engine::general_purpose::STANDARD.encode(format!("vcs:{t}")); clone.env("GIT_CONFIG_COUNT", "1").env("GIT_CONFIG_KEY_0", "http.extraHeader").env("GIT_CONFIG_VALUE_0", format!("Authorization: Basic {basic}")); } let status = clone .args(["clone", "--mirror", "--quiet", &url]) .arg(&temp) .stdin(Stdio::null()) .status() .map_err(|_| git_missing())?; if !status.success() { let _ = std::fs::remove_dir_all(&temp); return Err(Fail::new("not_found", format!("couldn't clone {url}")).state("nothing changed")); } Ok(Repo { dir: temp.clone(), temp: Some(temp) }) } impl Repo { /// The default branch's name and commit. fn default_branch(&self) -> Result<(String, String), Fail> { let head = git(&self.dir, &["symbolic-ref", "-q", "HEAD"]).unwrap_or_default(); let name = head.trim().strip_prefix("refs/heads/").unwrap_or("HEAD").to_string(); let sha = git(&self.dir, &["rev-parse", "--verify", "-q", "HEAD^{commit}"]) .map_err(|_| Fail::new("bad_usage", "the git repository has no commits").state("nothing changed"))?; Ok((name, sha.trim().to_string())) } } /// `git cat-file --batch`: objects by ID, through one long-running git. struct Cat { child: Child, stdin: ChildStdin, out: BufReader, } impl Cat { fn open(dir: &Path) -> Result { let mut child = Command::new("git") .arg("--git-dir") .arg(dir) .args(["cat-file", "--batch"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn() .map_err(|_| git_missing().into_anyhow())?; let stdin = child.stdin.take().unwrap(); let out = BufReader::with_capacity(1 << 20, child.stdout.take().unwrap()); Ok(Cat { child, stdin, out }) } fn get(&mut self, sha: &str) -> Result<(String, Vec)> { writeln!(self.stdin, "{sha}")?; self.stdin.flush()?; let mut header = String::new(); self.out.read_line(&mut header)?; let parts: Vec<&str> = header.trim_end().split(' ').collect(); if parts.len() != 3 { bail!("git object {sha} is missing"); } let size: usize = parts[2].parse()?; let mut data = vec![0; size]; self.out.read_exact(&mut data)?; let mut newline = [0u8; 1]; self.out.read_exact(&mut newline)?; Ok((parts[1].to_string(), data)) } } impl Drop for Cat { fn drop(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); } } fn hex(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } /// Git trees and files as vcs objects, each converted once. struct Converter<'a> { store: &'a Store, cat: Cat, trees: HashMap, Option>, files: HashMap, Id>, /// Bytes in a git object ID: 20 (SHA-1), or 32 in a SHA-256 repository. width: usize, submodules: usize, odd_names: usize, } impl<'a> Converter<'a> { fn new(store: &'a Store, repo: &Repo) -> Result> { Ok(Converter { store, cat: Cat::open(&repo.dir)?, trees: HashMap::new(), files: HashMap::new(), width: repo.id_width(), submodules: 0, odd_names: 0 }) } /// Entries of a git tree: (mode, name, id). fn entries(&mut self, sha: &[u8]) -> Result, Vec, Vec)>> { let (kind, data) = self.cat.get(&hex(sha))?; if kind != "tree" { bail!("git object {} isn't a tree", hex(sha)); } let mut out = Vec::new(); let mut at = 0; while at < data.len() { let space = data[at..].iter().position(|&b| b == b' ').ok_or_else(|| anyhow!("bad git tree"))? + at; let nul = data[space + 1..].iter().position(|&b| b == 0).ok_or_else(|| anyhow!("bad git tree"))? + space + 1; let id = data.get(nul + 1..nul + 1 + self.width).ok_or_else(|| anyhow!("bad git tree"))?.to_vec(); out.push((data[at..space].to_vec(), data[space + 1..nul].to_vec(), id)); at = nul + 1 + self.width; } Ok(out) } /// A git tree as a vcs tree (None: nothing left in it, for a subfolder). fn tree(&mut self, sha: Vec, root: bool) -> Result> { if let Some(t) = self.trees.get(&sha) { return Ok(*t); } let mut entries = Vec::new(); for (mode, name, id) in self.entries(&sha)? { let Ok(name) = String::from_utf8(name) else { self.odd_names += 1; continue; }; // vcs's or git's own folder anywhere, a backslash: unsafe to write out. if !tree::safe_name(&name) { self.odd_names += 1; continue; } let (kind, id) = match mode.as_slice() { b"40000" | b"040000" => match self.tree(id, false)? { Some(t) => (EntryKind::Tree, t), None => continue, }, // A submodule: the commit it points at, kept as it is. b"160000" => { self.submodules += 1; (EntryKind::Gitlink, self.store.put_file(hex(&id).as_bytes())?) } b"120000" => (EntryKind::Link, self.file(id)?), m if m.ends_with(b"755") => (EntryKind::Exec, self.file(id)?), _ => (EntryKind::File, self.file(id)?), }; entries.push(TreeEntry { name, kind, id }); } let out = if entries.is_empty() && !root { None } else { Some(self.store.put(&Object::new(Kind::Tree, tree::encode(&entries)))?) }; self.trees.insert(sha, out); Ok(out) } fn file(&mut self, sha: Vec) -> Result { if let Some(f) = self.files.get(&sha) { return Ok(*f); } let (_, data) = self.cat.get(&hex(&sha))?; let id = self.store.put_file(&data)?; self.files.insert(sha, id); Ok(id) } /// Native ID → git ID for everything under a converted git tree. fn ids_under(&mut self, sha: Vec, out: &mut Vec<(Id, String)>) -> Result<()> { let Some(Some(native)) = self.trees.get(&sha).copied() else { return Ok(()) }; out.push((native, hex(&sha))); for (mode, _, id) in self.entries(&sha)? { if mode == b"40000" || mode == b"040000" { self.ids_under(id, out)?; } else if mode == b"160000" { // Not a git object of this repository. } else if let Some(f) = self.files.get(&id) { out.push((*f, hex(&id))); } } Ok(()) } } /// A git object ID's bytes: 40 hex digits (SHA-1), or 64 (SHA-256). fn sha_bytes(text: &str) -> Result> { let t = text.trim(); if t.len() != 40 && t.len() != 64 { bail!("{t} isn't a git object ID"); } (0..t.len() / 2).map(|i| Ok(u8::from_str_radix(&t[i * 2..i * 2 + 2], 16)?)).collect() } /// A change ID made from a git commit, so importing twice gives the same IDs. fn change_id(sha: &str) -> String { sha_bytes(sha).map(|b| b.iter().take(ids::CHANGE_ID_LEN).map(|x| ids::LETTERS[*x as usize % ids::LETTERS.len()] as char).collect()).unwrap_or_else(|_| ids::new_change_id()) } struct Commit { sha: String, tree: String, author: String, time: i64, title: String, body: String, } fn split_message(message: &str) -> (String, String) { let mut parts = message.trim().splitn(2, '\n'); (parts.next().unwrap_or("").trim().to_string(), parts.next().unwrap_or("").trim().to_string()) } /// Commits on `rev`'s first-parent line, oldest first. fn commits(repo: &Repo, rev: &str, limit: Option) -> Result, Fail> { let mut args = vec!["log", "--first-parent", "--format=%H%x00%T%x00%an%x00%ae%x00%at%x00%B%x1e"]; let n; if let Some(l) = limit { n = format!("-{l}"); args.push(&n); } else { args.push("--reverse"); } args.push(rev); let text = git(&repo.dir, &args)?; let mut out = Vec::new(); for record in text.split('\x1e') { let record = record.trim_start_matches('\n'); if record.is_empty() { continue; } let f: Vec<&str> = record.splitn(6, '\0').collect(); if f.len() != 6 { continue; } let (title, body) = split_message(f[5]); out.push(Commit { sha: f[0].into(), tree: f[1].into(), author: format!("{} <{}>", f[2], f[3]), time: f[4].parse().unwrap_or(0), title, body }); } Ok(out) } fn progress(app: &App, text: &str, last: &mut Instant) { if !app.json && std::io::stderr().is_terminal() && last.elapsed().as_millis() > 500 { eprint!("\r{text} "); *last = Instant::now(); } } fn notes_for(app: &App, conv: &Converter) { if conv.submodules > 0 { app.notes.borrow_mut().push(format!("{} submodules kept as pointers (their folders stay empty here)", conv.submodules)); } if conv.odd_names > 0 { app.notes.borrow_mut().push(format!("{} paths were left out (names that aren't UTF-8, a .git or .vcs folder, or a backslash)", conv.odd_names)); } } /// `vcs import [folder] [--to host/owner/name]`: a new project with /// the repository's history. `--change ` instead brings one git branch /// into the current workspace's project as a change. pub fn import(app: &App, source: &str, to: Option, folder: Option, change: Option) -> Out { if let Some(rev) = change { return import_change(app, source, &rev); } import_as(app, source, to, folder, None) } /// An import whose clone uses `token` (a project just switched on in Upstream: its first vcs get). pub fn import_as(app: &App, source: &str, to: Option, folder: Option, token: Option<&str>) -> Out { let repo = open_as(app, source, token)?; let (branch, _) = repo.default_branch()?; let name = clone_url(source).trim_end_matches('/').rsplit('/').next().unwrap_or("project").trim_end_matches(".git").to_string(); let root = app.cwd.join(folder.unwrap_or_else(|| PathBuf::from(if name.is_empty() { "project".to_string() } else { name }))); crate::app::ensure_dir_empty(&root)?; let list = commits(&repo, "HEAD", None)?; let project = Project { id: format!("local-{}", ids::new_workspace_id()), address: String::new(), kind: "local".into(), api: None, checked: ids::now(), }; let mut conv = Converter::new(&app.store, &repo)?; // What version 1 starts from. app.store.put(&tree::empty())?; let mut versions = Vec::with_capacity(list.len()); let mut prev: Option = None; let mut last = Instant::now(); for (i, c) in list.iter().enumerate() { progress(app, &format!("reading git history: {} of {} commits", i + 1, list.len()), &mut last); let tree = conv.tree(sha_bytes(&c.tree)?, true)?.unwrap_or_else(tree::empty_id); let v = Version { n: i as u64 + 1, tree, prev, change: change_id(&c.sha), author: c.author.clone(), time: c.time, title: c.title.clone(), body: c.body.clone(), git: Some(c.sha.clone()), }; prev = Some(v.id(&project.id)); versions.push(v); } if !app.json && std::io::stderr().is_terminal() { eprint!("\r{}\r", " ".repeat(60)); } let main = versions.last().cloned().ok_or_else(|| Fail::new("bad_usage", "the git repository has no commits").state("nothing changed"))?; let mut git_ids = Vec::new(); conv.ids_under(sha_bytes(&list.last().unwrap().tree)?, &mut git_ids)?; notes_for(app, &conv); // The project, its history and a workspace on main. std::fs::create_dir_all(&root)?; let root = crate::sys::canonical(&root)?; let project = Project { address: root.to_string_lossy().into_owned(), ..project }; app.db.put_project(&project)?; { let tx = app.db.conn.unchecked_transaction()?; for v in &versions { app.db.put_version(&project.id, v)?; } tx.commit()?; } app.db.put_git_ids(&project.id, &git_ids)?; let origin = if app.cwd.join(source).exists() { crate::sys::canonical(&app.cwd.join(source))?.to_string_lossy().into_owned() } else { clone_url(source) }; app.db.set_import(&project.id, &origin, &branch)?; let author = app.author(&project); let c = Change::new(ids::new_change_id(), &project.id, main.tree, Some(main.n), &author); app.db.put_change(&c)?; let workspace = Workspace { id: ids::new_workspace_id(), project: project.id.clone(), path: root.clone(), change: c.id.clone(), only: vec![] }; ws::write_id(&root, &workspace.id)?; let files = app.files(&main.tree)?; let mut index = ws::materialize(&root, &app.store, &Index::default(), &files, &[], &[], None)?; index.save(&root)?; app.db.put_workspace(&workspace)?; drop(conv); drop(repo); let mut r = Report::new(); r.line("imported", format!("{} commits from {branch} as versions 1 to {}", versions.len(), main.n)) .line("workspace", root.display().to_string()) .set("project", project.address.clone()) .set("workspace", root.display().to_string()) .set("versions", versions.len()) .set("main", main.n); if let Some(target) = to { let cwd_app = App::open(app.json, root.clone()).map_err(Fail::from)?; let published = crate::remote::publish(&cwd_app, &target)?; for note in cwd_app.notes.borrow().iter() { app.notes.borrow_mut().push(note.clone()); } r.line("published", target.clone()).set("published", serde_json::Value::Object(published.json.clone())); r.next("keep working; vcs land lands on the host"); } else { r.next(format!("cd {} and keep working; vcs publish puts it on a host", root.display())); } Ok(r) } /// `vcs import --change `: the git branch `rev`, as a new change /// in this workspace. It starts from where the branch left main. fn import_change(app: &App, source: &str, rev: &str) -> Out { let mut s = app.session()?; let current = app.save(&mut s)?; let repo = open(app, source)?; let tip = git(&repo.dir, &["rev-parse", "--verify", "-q", &format!("{rev}^{{commit}}")]) .map_err(|_| Fail::new("not_found", format!("{rev} isn't in {source}")).state("nothing changed"))? .trim() .to_string(); // Where the branch left vcs: the newest commit on its first-parent line // that is a version, named in the commit (the git view writes // Vcs-Version) or read by an import. app.refresh(&s.project)?; // Git commits are matched to versions by their git IDs, so all of // main's history is needed here. app.history(&s.project, 1)?; let line = commits(&repo, &tip, Some(100_000))?; let mut found = None; for c in &line { let named = c.body.lines().rev().find_map(|l| l.strip_prefix("Vcs-Version: ").and_then(|n| n.trim().parse::().ok())); let v = match named { Some(n) => app.db.version(&s.project.id, n)?, None => app.db.version_by_git(&s.project.id, &c.sha)?, }; if let Some(v) = v { found = Some((c, v)); break; } } let mut conv = Converter::new(&app.store, &repo)?; if found.is_none() { // No commit is one the project knows by ID (another copy of the // history, a SHA-256 repository): a commit whose files are a version's. for c in line.iter().take(1000) { let tree = conv.tree(sha_bytes(&c.tree)?, true)?.unwrap_or_else(tree::empty_id); if let Some(v) = app.db.version_by_tree(&s.project.id, &tree)? { found = Some((c, v)); break; } } } let (base_commit, version) = found.ok_or_else(|| { Fail::new("not_found", format!("{rev} doesn't start from any version of this project")) .state("nothing changed") .next("vcs import --change works on branches of this project's git view") })?; let tip_commit = line.first().ok_or_else(|| Fail::new("failed", "couldn't read the git history"))?; // The change's base is the git view's tree (without private paths), so // landing keeps whatever the view left out. let base_tree = conv.tree(sha_bytes(&base_commit.tree)?, true)?.unwrap_or_else(tree::empty_id); let head = conv.tree(sha_bytes(&tip_commit.tree)?, true)?.unwrap_or_else(tree::empty_id); notes_for(app, &conv); drop(conv); if head == base_tree { return Err(Fail::new("empty_change", format!("{rev} changes nothing since version {}", version.n)).state("nothing changed")); } let mut c = Change::new(ids::new_change_id(), &s.project.id, base_tree, Some(version.n), &app.author(&s.project)); let (title, body) = (tip_commit.title.clone(), tip_commit.body.clone()); c.title = title; c.body = body; c.head_tree = Some(head); c.saves = 1; app.db.put_change(&c)?; app.db.add_save(&Save { change: c.id.clone(), n: 1, tree: head, base_tree, at: ids::now(), uploaded: false }, &s.ws.id)?; app.show_tree(&s, &head)?; app.put_workspace_change(&mut s, &c.id)?; app.db.add_op(&s.ws.id, "new", &json!({ "from": current.id, "to": c.id }))?; let _ = app.host(&s.project).publish_change(&app.db, &app.store, &s.project, &c, &s.ws.id); let mut r = Report::new(); r.line("change", format!("{} \"{}\", from git {} on version {}", app.short(&s.project, &c.id), c.title, &tip[..12], version.n)); r.line("kept", format!("{} is saved; vcs switch goes back to it", app.short(&s.project, ¤t.id))); r.set("change", json!({ "id": c.id, "title": c.title, "base_version": version.n, "git": tip })); r.next("vcs land"); Ok(r) } /// Copies an imported project's original git history to the host's git /// view, so the view carries on from the same commits. Nothing happens if /// the view's main is already there; a view with other history is left as /// it is (the host pauses the view and says so). pub fn push_history(app: &App, project: &Project, git_url: &str, token: &str) -> Result, Fail> { let Some((source, branch)) = app.db.import_source(&project.id)? else { return Ok(None) }; // The newest version that came from git (later ones are native). let Some(last) = app.db.versions(&project.id, i64::MAX)?.into_iter().find_map(|v| v.git) else { return Ok(None) }; let auth = format!("Authorization: Basic {}", base64::engine::general_purpose::STANDARD.encode(format!("vcs:{token}"))); // The token goes in git's environment, not its arguments. let with_auth = |cmd: &mut Command| { cmd.env("GIT_CONFIG_COUNT", "1").env("GIT_CONFIG_KEY_0", "http.extraHeader").env("GIT_CONFIG_VALUE_0", &auth).env("GIT_TERMINAL_PROMPT", "0"); }; let mut ls = Command::new("git"); ls.args(["ls-remote", git_url, "refs/heads/main"]).stdin(Stdio::null()).stderr(Stdio::null()); with_auth(&mut ls); let out = ls.output().map_err(|_| git_missing())?; if out.status.success() && String::from_utf8_lossy(&out.stdout).split_whitespace().next() == Some(last.as_str()) { return Ok(None); } let repo = open(app, &source)?; // Every branch and tag as it is, and main at the last imported commit. let _ = branch; let mut refspecs = vec!["refs/tags/*:refs/tags/*".to_string(), format!("{last}:refs/heads/main")]; for name in git(&repo.dir, &["for-each-ref", "--format=%(refname)", "refs/heads"])?.lines() { if name != "refs/heads/main" && !name.is_empty() { refspecs.push(format!("{name}:{name}")); } } let mut push = Command::new("git"); push.arg("--git-dir").arg(&repo.dir).args(["push", "--quiet", git_url]).args(&refspecs).stdin(Stdio::null()); with_auth(&mut push); let out = push.output().map_err(|_| git_missing())?; if !out.status.success() { return Ok(Some(format!( "the git view at {git_url} already has other history, so it stays paused: {}", String::from_utf8_lossy(&out.stderr).lines().last().unwrap_or("").trim() ))); } Ok(Some(format!("copied the git history to {git_url}"))) }