pig2014/honey-biscuit-workshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: replace internal fetch utilities with ResourceRegistry

Browse Source 
Remove the monolithic utils::fetch module (HTTP download, git clone,
checksum, extraction) and utils::fetch_plugin module from the baker.
Introduce a ResourceRegistry type to centrally track pipeline
resources by name and path.

The actual fetching logic is now delegated to the workshop-getterurl
crate, which gains optional md5 and sha1 hash support behind default
features. Also flatten the builtin module in workshop-getterurl.
This commit is contained in:
Catty Steve
2026-05-19 23:22:59 +08:00
parent 6e7b96cd66
commit 28abc5d207
22 changed files with 249 additions and 957 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workshop-baker/src/bare.rs
+3 -2
View File
@@ -2,9 +2,9 @@ use std::path::PathBuf;
use crate::cli::{BareCommands, Cli};
use crate::error::CliError;
use crate::types::resource::ResourceRegistry;
use crate::{bake::bake, finalize::finalize, prebake::prebake};
use workshop_engine::{EventSender, ExecutionContext};
fn or_workshop(path: &Option<PathBuf>, name: &str) -> PathBuf {
path
.clone()
@@ -18,6 +18,7 @@ pub async fn run_bare(
event_tx: EventSender,
pipeline: &str,
build_id: &str,
registry: &ResourceRegistry,
) -> Result<(), CliError> {
let (stage, config) = match command {
BareCommands::Prebake { .. } => ("prebake", or_workshop(&cli.prebake, "prebake.yml")),
@@ -42,7 +43,7 @@ pub async fn run_bare(
bake(&config, &bake_base, &prebake_path, cli, ctx, event_tx).await?;
}
BareCommands::Finalize { .. } => {
finalize(&config, cli, ctx, event_tx).await?;
finalize(&config, cli, ctx, event_tx, registry).await?;
}
}
Ok(())
workshop-baker/src/finalize.rs
+4 -28
View File
@@ -8,12 +8,10 @@ pub mod template;
pub mod types;
use crate::cli::Cli;
use crate::finalize::constant::FINALIZE_STANDALONE_PLUGIN_PATH;
use crate::utils::fetch_plugin::{self, FetchArgument};
use crate::types::resource::ResourceRegistry;
pub use config::*;
pub use error::FinalizeError;
use std::path::Path;
use std::path::PathBuf;
use workshop_engine::EventSender;
pub fn parse(path: &Path) -> Result<FinalizeConfig, FinalizeError> {
@@ -27,6 +25,7 @@ pub async fn finalize(
_cli: &Cli,
ctx: &mut workshop_engine::ExecutionContext,
event_tx: EventSender,
registry: &ResourceRegistry,
) -> Result<(), FinalizeError> {
let mut finalize = parse(finalize_path)?;
let validate_result = finalize.validate();
@@ -36,33 +35,10 @@ pub async fn finalize(
}
return Err(FinalizeError::ValidateError(errors));
};
if ctx.standalone {
// In standalone mode, finalize should download plugin by itself
log::debug!("Downloading {} plugins", finalize.plugin.len());
let plugin_path = PathBuf::from(FINALIZE_STANDALONE_PLUGIN_PATH);
// if plugin_path.exists() {
// std::fs::remove_dir_all(&plugin_path).map_err(FinalizeError::IoError)?;
// }
for (name, data) in finalize.plugin.iter_mut() {
log::info!("Downloading plugin {}", name);
let (source, config) = data.iter_mut().next().unwrap_or_else(|| {
panic!(
"Internal Error: Plugin {} has no source configured, this should have been caught by validation!",
name
)
});
let fetch_argument = FetchArgument {
checksum: config.checksum.clone(),
shallow: config.shallow,
};
fetch_plugin::fetch_plugin(source, name, &plugin_path, fetch_argument).await?;
config.runtime.fspath = Some(plugin_path.join(name));
}
}
// Register Plugins
// Register Plugins (paths resolved from registry)
log::info!("Registering {} plugins", finalize.plugin.len());
let registered_plugins = plugin::register(finalize.plugin, None, &mut finalize.notification)?;
let registered_plugins = plugin::register(finalize.plugin, None, &mut finalize.notification, registry)?;
// Early hook
let earlyhook_result = stage::hook::hook(
workshop-baker/src/finalize/error.rs
-7
View File
@@ -1,4 +1,3 @@
use crate::utils::fetch::FetchError;
use std::path::PathBuf;
use thiserror::Error;
@@ -56,12 +55,6 @@ impl HasExitCode for FinalizeError {
}
}
impl From<FetchError> for FinalizeError {
fn from(e: FetchError) -> Self {
FinalizeError::PluginError(PluginError::GeneralError(e.to_string()))
}
}
#[derive(Error, Debug)]
pub enum PluginError {
#[error("Unknown plugin: {0}")]
workshop-baker/src/finalize/plugin.rs
+10 -1
View File
@@ -3,6 +3,7 @@ use crate::finalize::constant::{FINALIZE_PLUGIN_MANIFEST, FINALIZE_PLUGIN_MANIFE
use crate::finalize::plugin::metadata::{PluginMetadata, PluginType};
use workshop_engine::{EventSender, ExecutionContext};
use crate::{ finalize::error::PluginError};
use crate::types::resource::ResourceRegistry;
use async_trait::async_trait;
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
@@ -215,6 +216,7 @@ pub fn register(
external_plugins: RawPluginMap,
list: Option<&HashSet<String>>,
notification_config: &mut crate::finalize::NotificationConfig,
registry: &ResourceRegistry,
) -> Result<PluginMap, PluginError> {
let mut plugins: PluginMap = HashMap::new();
let mut count = 0;
@@ -256,7 +258,14 @@ pub fn register(
log::warn!("Plugin {} is not available: {}", name, reason);
continue;
}
let metadata = read_manifest(config.runtime.fspath, &name)?;
// Resolve plugin path from registry instead of config.runtime.fspath
let plugin_path = registry.resolve(&name).ok_or_else(|| {
PluginError::PluginUnavailable {
name: name.clone(),
reason: "plugin not found in resource registry".to_string(),
}
})?;
let metadata = read_manifest(Some(plugin_path), &name)?;
let plugin_type = get_plugin_type(metadata.plugin_type.clone(), &metadata.entrypoint);
let entry = get_entry(plugin_type, &name)?;
plugins.insert(
workshop-baker/src/main.rs
+10 -3
View File
@@ -1,3 +1,4 @@
use workshop_baker::types::resource::ResourceRegistry;
use std::path::PathBuf;
use workshop_baker::{
@@ -48,12 +49,17 @@ async fn worker_main(cli: Cli) -> Result<(), CliError> {
let (to_tx, to_rx) = tokio::sync::mpsc::channel(256);
let (retry_tx, retry_rx) = tokio::sync::mpsc::channel(256);
// Resource registry — populated before stages, consumed by stages
let registry = ResourceRegistry::new();
// TODO: pre-fetch plugins and templates into registry
let notify_path = finalize_path.clone();
let notify_ctx = ctx.clone();
let notify_etx = event_tx.clone();
let debug = cli.debug.clone();
let notify_registry = registry.clone();
let notify_handle = tokio::spawn(async move {
notify(&notify_path, &mut notify_ctx.clone(), notify_etx, to_rx, retry_tx, debug).await
notify(&notify_path, &mut notify_ctx.clone(), notify_etx, to_rx, retry_tx, debug, &notify_registry).await
});
let nevent_tx = to_tx.clone();
@@ -64,7 +70,7 @@ async fn worker_main(cli: Cli) -> Result<(), CliError> {
let bake_base_path = or_workshop(&cli.bake_base, "bake_base.sh");
let pb_path = or_workshop(&cli.prebake, "prebake.yml");
bake(&bake_path, &bake_base_path, &pb_path, &cli, &mut ctx, event_tx.clone()).await?;
finalize(&finalize_path, &cli, &mut ctx, event_tx.clone()).await?;
finalize(&finalize_path, &cli, &mut ctx, event_tx.clone(), &registry).await?;
Ok(())
}.await;
@@ -109,7 +115,8 @@ async fn main() {
event_rx, pipeline.clone(), build_id.clone(),
));
let result = run_bare(&cli, command, &mut ctx, event_tx, &pipeline, &build_id).await;
let registry = ResourceRegistry::new();
let result = run_bare(&cli, command, &mut ctx, event_tx, &pipeline, &build_id, &registry).await;
drop(ctx);
let _ = event_handle.await;
workshop-baker/src/notify.rs
+14 -36
View File
@@ -2,10 +2,9 @@ use crate::finalize::FinalizeConfig;
use crate::finalize::NotificationConfig;
use crate::finalize::NotificationTemplate;
use crate::finalize::config::TriggerItemRaw;
use crate::finalize::constant::FINALIZE_STANDALONE_PLUGIN_PATH;
use crate::types::resource::ResourceRegistry;
use crate::finalize::parse;
use crate::finalize::plugin;
use crate::utils::fetch_plugin;
use crate::notify::types::Matchable;
use crate::notify::types::Method;
use crate::notify::types::MethodStatus;
@@ -31,7 +30,6 @@ use chrono::Utc;
use globset::{Glob, GlobSetBuilder};
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::PathBuf;
use std::path::Path;
mod error;
@@ -72,6 +70,7 @@ pub async fn notify(
mut rx: NotificationEventReceiver,
retry_tx: NotificationEventSender,
debug: Option<String>,
registry: &ResourceRegistry,
) -> anyhow::Result<()> {
let mut finalize = parse(finalize_path)?;
validate(&finalize)?;
@@ -101,44 +100,23 @@ pub async fn notify(
}
}
if ctx.standalone {
fetch_plugin::fetch_plugins(
&PathBuf::from(FINALIZE_STANDALONE_PLUGIN_PATH),
&mut finalize.plugin,
Some(&plugin_list),
)
.await?;
}
let notify_plugins = plugin::register(finalize.plugin, Some(&plugin_list), &mut finalize.notification)?;
let notify_plugins = plugin::register(finalize.plugin, Some(&plugin_list), &mut finalize.notification, registry)?;
// Fetch external templates referenced by use_ (git:// / https:// URLs)
let template_cache = std::env::temp_dir().join("workshop-templates");
// Resolve external template references from registry (pre-fetched)
for (name, tmpl_def) in finalize.notification.templates.iter_mut() {
if let Some(ref url) = tmpl_def.use_ {
if url.contains("://") {
// External fetch
match crate::utils::fetch::fetch_resource(name, url, &template_cache).await {
Ok(path) => {
// Read the fetched template file
let content = std::fs::read_to_string(path.join("template.yml"))
.or_else(|_| std::fs::read_to_string(path.join("template.yaml")));
if let Ok(content) = content {
if let Ok(fetched) = serde_yaml::from_str::<crate::finalize::NotificationTemplateDef>(&content) {
tmpl_def.on_success = fetched.on_success.or(tmpl_def.on_success.take());
tmpl_def.on_failure = fetched.on_failure.or(tmpl_def.on_failure.take());
tmpl_def.on_finish_of = fetched.on_finish_of.or(tmpl_def.on_finish_of.take());
tmpl_def.use_ = None; // Resolved
}
}
}
Err(e) => {
log::error!("Failed to fetch template '{}' from '{}': {}", name, url, e);
if tmpl_def.use_.is_some() {
if let Some(path) = registry.resolve(name) {
let content = std::fs::read_to_string(path.join("template.yml"))
.or_else(|_| std::fs::read_to_string(path.join("template.yaml")));
if let Ok(content) = content {
if let Ok(fetched) = serde_yaml::from_str::<crate::finalize::NotificationTemplateDef>(&content) {
tmpl_def.on_success = fetched.on_success.or(tmpl_def.on_success.take());
tmpl_def.on_failure = fetched.on_failure.or(tmpl_def.on_failure.take());
tmpl_def.on_finish_of = fetched.on_finish_of.or(tmpl_def.on_finish_of.take());
tmpl_def.use_ = None; // Resolved
}
}
}
// Non-:// use_ values are local template references → handled in Part 2
}
}
workshop-baker/src/types.rs
+1
View File
@@ -1 +1,2 @@
pub mod buildstatus;
pub mod resource;
workshop-baker/src/types/resource.rs
+119
View File
@@ -0,0 +1,119 @@
//! Internal resource representation for the baker loop.
//!
//! A [`Resource`] is the local intermediate state of a fetched external asset.
//! It carries a logical name and a filesystem path. Resources are registered
//! into a [`ResourceRegistry`] before the pipeline starts and consumed
//! exclusively through `resolve()` — the registry is treated as read-only
//! once the pipeline is running.
use std::collections::HashMap;
use std::path::PathBuf;
/// A locally resolved resource with a logical name and filesystem path.
///
/// Created by the resource fetcher (bakerd in worker mode, or the baker
/// itself in standalone mode) and consumed by pipeline stages via the
/// [`ResourceRegistry`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Resource {
/// Logical name used for lookup (e.g., `"plugin/docker-push"`).
pub name: String,
/// Local filesystem path to the fetched resource.
pub path: PathBuf,
}
impl Resource {
/// Create a new resource entry.
pub fn new(name: impl Into<String>, path: PathBuf) -> Self {
Resource {
name: name.into(),
path,
}
}
}
/// A read-mostly registry of resources available to the pipeline.
///
/// Populated during the fetch phase (before prebake) and then treated as
/// immutable for the remainder of the pipeline.
#[derive(Debug, Clone, Default)]
pub struct ResourceRegistry {
map: HashMap<String, Resource>,
}
impl ResourceRegistry {
/// Create an empty registry.
pub fn new() -> Self {
ResourceRegistry {
map: HashMap::new(),
}
}
/// Register a resource under its logical name.
///
/// Overwrites any existing entry with the same name.
pub fn register(&mut self, name: impl Into<String>, path: PathBuf) {
let name = name.into();
self.map.insert(name.clone(), Resource::new(name, path));
}
/// Resolve a logical name to its filesystem path.
///
/// Returns `None` if the resource has not been registered.
pub fn resolve(&self, name: &str) -> Option<PathBuf> {
self.map.get(name).map(|r| r.path.clone())
}
/// Look up a resource entry (includes both name and path).
pub fn get(&self, name: &str) -> Option<&Resource> {
self.map.get(name)
}
/// Returns the number of registered resources.
pub fn len(&self) -> usize {
self.map.len()
}
/// Returns `true` if no resources are registered.
pub fn is_empty(&self) -> bool {
self.map.is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_register_and_resolve() {
let mut reg = ResourceRegistry::new();
reg.register("plugin/x", PathBuf::from("/cache/plugin_x"));
assert_eq!(
reg.resolve("plugin/x"),
Some(PathBuf::from("/cache/plugin_x"))
);
}
#[test]
fn test_resolve_missing() {
let reg = ResourceRegistry::new();
assert_eq!(reg.resolve("nonexistent"), None);
}
#[test]
fn test_overwrite() {
let mut reg = ResourceRegistry::new();
reg.register("a", PathBuf::from("/first"));
reg.register("a", PathBuf::from("/second"));
assert_eq!(reg.resolve("a"), Some(PathBuf::from("/second")));
}
#[test]
fn test_len() {
let mut reg = ResourceRegistry::new();
assert!(reg.is_empty());
reg.register("a", PathBuf::from("/a"));
reg.register("b", PathBuf::from("/b"));
assert_eq!(reg.len(), 2);
}
}
workshop-baker/src/utils.rs
-2
View File
@@ -1,2 +0,0 @@
pub mod fetch;
pub mod fetch_plugin;
workshop-baker/src/utils/fetch/checksum.rs
-137
View File
@@ -1,137 +0,0 @@
// Code in this module PARTIALLY OR FULLY utilized AI Coding Agent.
use super::types::ChecksumType;
use sha2::{Digest, Sha256, Sha512};
/// Detects the checksum type based on the length of the checksum string.
///
/// Returns `Ok(None)` for empty string (checksum disabled).
/// Returns `Ok(Some(ChecksumType))` for valid SHA256 (64 chars) or SHA512 (128 chars).
/// Returns `Err` for invalid lengths (32=MD5, 40=SHA1) or unsupported lengths.
pub fn detect_checksum_type(checksum: &str) -> Result<Option<ChecksumType>, String> {
match checksum.len() {
0 => Ok(None),
32 => Err(format!(
"MD5 (first 8 chars: {}) is not supported for security reasons. Use SHA256 (64 chars) or SHA512 (128 chars)",
&checksum[..8.min(checksum.len())]
)),
40 => Err(format!(
"SHA1 (first 8 chars: {}) is deprecated and not supported. Use SHA256 (64 chars) or SHA512 (128 chars)",
&checksum[..8.min(checksum.len())]
)),
64 => Ok(Some(ChecksumType::Sha256)),
128 => Ok(Some(ChecksumType::Sha512)),
_ => Err(format!(
"Invalid checksum length: {}. Expected 0 (disabled), 64 (SHA256), or 128 (SHA512). First 8 chars: {}",
checksum.len(),
&checksum[..8.min(checksum.len())]
)),
}
}
/// Verifies that the data matches the expected checksum.
///
/// Compares the computed hash of `data` against `expected` using the specified `checksum_type`.
/// Returns `Ok(())` if the checksums match, `Err` otherwise.
pub fn verify_checksum(
data: &[u8],
expected: &str,
checksum_type: ChecksumType,
) -> Result<(), String> {
let actual = match checksum_type {
ChecksumType::Sha256 => {
let mut hasher = Sha256::new();
hasher.update(data);
format!("{:x}", hasher.finalize())
}
ChecksumType::Sha512 => {
let mut hasher = Sha512::new();
hasher.update(data);
format!("{:x}", hasher.finalize())
}
};
if actual.eq_ignore_ascii_case(expected) {
Ok(())
} else {
Err(format!(
"Checksum mismatch: expected {}..., got {}...",
&expected[..8.min(expected.len())],
&actual[..8.min(actual.len())]
))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_detect_checksum_type_empty() {
assert_eq!(detect_checksum_type("").unwrap(), None);
}
#[test]
fn test_detect_checksum_type_md5_rejected() {
let result = detect_checksum_type("a".repeat(32).as_str());
assert!(result.is_err());
assert!(result.unwrap_err().contains("MD5"));
}
#[test]
fn test_detect_checksum_type_sha1_rejected() {
let result = detect_checksum_type("a".repeat(40).as_str());
assert!(result.is_err());
assert!(result.unwrap_err().contains("SHA1"));
}
#[test]
fn test_detect_checksum_type_sha256() {
assert_eq!(
detect_checksum_type("a".repeat(64).as_str()).unwrap(),
Some(ChecksumType::Sha256)
);
}
#[test]
fn test_detect_checksum_type_sha512() {
assert_eq!(
detect_checksum_type("a".repeat(128).as_str()).unwrap(),
Some(ChecksumType::Sha512)
);
}
#[test]
fn test_detect_checksum_type_invalid_length() {
let result = detect_checksum_type("a".repeat(50).as_str());
assert!(result.is_err());
assert!(result.unwrap_err().contains("Invalid checksum length"));
}
#[test]
fn test_verify_checksum_sha256_valid() {
let data = b"hello";
let expected = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
assert!(verify_checksum(data, expected, ChecksumType::Sha256).is_ok());
}
#[test]
fn test_verify_checksum_sha256_invalid() {
let data = b"hello";
let expected = "0000000000000000000000000000000000000000000000000000000000000000";
assert!(verify_checksum(data, expected, ChecksumType::Sha256).is_err());
}
#[test]
fn test_verify_checksum_sha512_valid() {
let data = b"hello";
let expected = "9b71d224bd62f3785d96d46ad3ea3d73319bfbc2890caadae2dff72519673ca72323c3d99ba5c11d7c7acc6e14b8c5da0c4663475c2e5c3adef46f73bcdec043";
assert!(verify_checksum(data, expected, ChecksumType::Sha512).is_ok());
}
#[test]
fn test_verify_checksum_sha512_invalid() {
let data = b"hello";
let expected = "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000";
assert!(verify_checksum(data, expected, ChecksumType::Sha512).is_err());
}
}
workshop-baker/src/utils/fetch/error.rs
-23
View File
@@ -1,23 +0,0 @@
use thiserror::Error;
#[derive(Error, Debug)]
pub enum FetchError {
#[error("Fetch failed for {name} ({url}): {reason}")]
FetchFailed {
name: String,
url: String,
reason: String,
},
#[error("Unknown URL scheme: {0}")]
UnknownScheme(String),
#[error("Invalid file path: {0}")]
InvalidPath(String),
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("General error: {0}")]
General(String),
}
workshop-baker/src/utils/fetch/extract.rs
-169
View File
@@ -1,169 +0,0 @@
// Code in this module PARTIALLY OR FULLY utilized AI Coding Agent.
use super::error::FetchError;
use std::path::{Path, PathBuf};
use tokio::task;
/// Compression type for tar archives.
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum CompressionType {
/// Gzip compressed (.tar.gz, .tgz)
Gzip,
/// Zstd compressed (.tar.zst, .tar.zstd, .tzst)
Zstd,
/// Uncompressed tar (.tar)
None,
}
impl CompressionType {
/// Detects compression type from file path extension.
///
/// Supports: .tar.gz, .tgz (Gzip), .tar.zst, .tar.zstd, .tzst (Zstd), .tar (None).
pub fn from_path(path: &std::path::Path) -> Result<Self, String> {
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("")
.to_lowercase();
if name.ends_with(".tar.gz") || name.ends_with(".tgz") {
Ok(CompressionType::Gzip)
} else if name.ends_with(".tar.zst")
|| name.ends_with(".tar.zstd")
|| name.ends_with(".tzst")
{
Ok(CompressionType::Zstd)
} else if name.ends_with(".tar") {
Ok(CompressionType::None)
} else {
Err(format!(
"Unsupported archive format: '{}'. Supported formats: .tar, .tar.gz, .tgz, .tar.zst, .tar.zstd, .tzst",
name
))
}
}
}
/// Extracts a tar archive to the destination directory.
///
/// Removes existing destination directory if present, creates a fresh directory,
/// then unpacks the archive using the appropriate decompression based on `compression`.
pub async fn extract_tar(
archive: &Path,
destdir: &PathBuf,
compression: CompressionType,
) -> Result<(), FetchError> {
if destdir.exists() {
tokio::fs::remove_dir_all(destdir).await.map_err(|e| {
FetchError::General(format!("Failed to remove existing directory: {}", e))
})?;
}
tokio::fs::create_dir_all(destdir)
.await
.map_err(|e| FetchError::General(format!("Failed to create directory: {}", e)))?;
let archive = archive.to_path_buf();
let destdir = destdir.clone();
task::spawn_blocking(move || {
let file = std::fs::File::open(&archive)
.map_err(|e| FetchError::General(format!("Failed to open archive: {}", e)))?;
match compression {
CompressionType::Gzip => {
let decoder = flate2::read::GzDecoder::new(file);
let mut archive = tar::Archive::new(decoder);
archive.unpack(&destdir).map_err(|e| {
FetchError::General(format!("Failed to unpack archive: {}", e))
})?;
}
CompressionType::Zstd => {
let decoder = zstd::stream::read::Decoder::new(file).map_err(|e| {
FetchError::General(format!("Failed to create zstd decoder: {}", e))
})?;
let mut archive = tar::Archive::new(decoder);
archive.unpack(&destdir).map_err(|e| {
FetchError::General(format!("Failed to unpack archive: {}", e))
})?;
}
CompressionType::None => {
let mut archive = tar::Archive::new(file);
archive.unpack(&destdir).map_err(|e| {
FetchError::General(format!("Failed to unpack archive: {}", e))
})?;
}
};
Ok::<(), FetchError>(())
})
.await
.map_err(|e| FetchError::General(format!("Task join error: {}", e)))??;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::Path;
#[test]
fn test_from_path_tar_gz() {
assert_eq!(
CompressionType::from_path(Path::new("archive.tar.gz")).unwrap(),
CompressionType::Gzip
);
}
#[test]
fn test_from_path_tgz() {
assert_eq!(
CompressionType::from_path(Path::new("archive.tgz")).unwrap(),
CompressionType::Gzip
);
}
#[test]
fn test_from_path_tar_zst() {
assert_eq!(
CompressionType::from_path(Path::new("archive.tar.zst")).unwrap(),
CompressionType::Zstd
);
}
#[test]
fn test_from_path_tar_zstd() {
assert_eq!(
CompressionType::from_path(Path::new("archive.tar.zstd")).unwrap(),
CompressionType::Zstd
);
}
#[test]
fn test_from_path_tzst() {
assert_eq!(
CompressionType::from_path(Path::new("archive.tzst")).unwrap(),
CompressionType::Zstd
);
}
#[test]
fn test_from_path_tar() {
assert_eq!(
CompressionType::from_path(Path::new("archive.tar")).unwrap(),
CompressionType::None
);
}
#[test]
fn test_from_path_unsupported() {
assert!(CompressionType::from_path(Path::new("archive.zip")).is_err());
}
#[test]
fn test_from_path_case_insensitive() {
assert_eq!(
CompressionType::from_path(Path::new("archive.TAR.GZ")).unwrap(),
CompressionType::Gzip
);
}
}
workshop-baker/src/utils/fetch/file.rs
-46
View File
@@ -1,46 +0,0 @@
use std::path::{Path, PathBuf};
use super::error::FetchError;
/// Copy a local file (resolved from file://) to the target directory.
/// Uses std::fs::copy, NOT symlink.
pub async fn fetch_file(path: &Path, name: &str, target: &Path) -> Result<PathBuf, FetchError> {
if !path.exists() {
return Err(FetchError::InvalidPath(path.display().to_string()));
}
std::fs::create_dir_all(target).map_err(FetchError::Io)?;
let dest = if path.is_dir() {
let dest = target.join(name);
copy_dir_all(path, &dest).map_err(FetchError::Io)?;
dest
} else {
let dest = target
.join(name)
.join(path.file_name().ok_or(FetchError::InvalidPath(
"no filename".into(),
))?);
std::fs::create_dir_all(dest.parent().unwrap_or(target)).map_err(FetchError::Io)?;
std::fs::copy(path, &dest).map_err(FetchError::Io)?;
dest
};
Ok(dest)
}
fn copy_dir_all(src: &Path, dst: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(dst)?;
for entry in std::fs::read_dir(src)? {
let entry = entry?;
let ty = entry.file_type()?;
let src_path = entry.path();
let dst_path = dst.join(entry.file_name());
if ty.is_dir() {
copy_dir_all(&src_path, &dst_path)?;
} else {
std::fs::copy(&src_path, &dst_path)?;
}
}
Ok(())
}
workshop-baker/src/utils/fetch/git.rs
-185
View File
@@ -1,185 +0,0 @@
// Code in this module PARTIALLY OR FULLY utilized AI Coding Agent.
use super::error::FetchError;
use std::path::Path;
use tokio::process::Command;
/// Version specification for a git repository.
#[derive(Debug, PartialEq)]
pub enum GitVersion {
/// No version specified (invalid for fetch operations)
None,
/// Single version string (tag or commit hash)
Single(String),
/// Mixed: try tag first, fallback to commit hash
Mixed(String, String),
}
/// Parses a git URL with optional version specification.
///
/// Format: `url` (no version), `url@tag` (single version), `url@tag:commit` (mixed version).
/// Returns (base_url, GitVersion).
pub fn parse_git_url(url: &str) -> (String, GitVersion) {
if let Some((base, version)) = url.rsplit_once('@') {
if version.contains(":") {
let (tag, commit) = version.split_once(':').unwrap();
return (
base.to_string(),
GitVersion::Mixed(tag.to_string(), commit.to_string()),
);
}
return (base.to_string(), GitVersion::Single(version.to_string()));
}
(url.to_string(), GitVersion::None)
}
/// Clones a git repository and checks out the specified version.
///
/// Removes existing destination directory, performs shallow or full clone,
/// then checks out the given version (tag or commit).
pub async fn git_clone_and_checkout(
baseurl: &str,
version: &str,
dest: &Path,
name: &str,
shallow: bool,
) -> Result<(), FetchError> {
if dest.exists() {
tokio::fs::remove_dir_all(dest)
.await
.map_err(|e| FetchError::FetchFailed {
name: name.to_string(),
url: baseurl.to_string(),
reason: format!("Failed to remove existing directory: {}", e),
})?;
}
let mut cmd = Command::new("git");
cmd.arg("clone");
if shallow {
cmd.arg("--depth").arg("1");
}
cmd.arg(baseurl).arg(dest);
let output = cmd.output().await.map_err(|e| FetchError::FetchFailed {
name: name.to_string(),
url: baseurl.to_string(),
reason: e.to_string(),
})?;
if !output.status.success() {
return Err(FetchError::FetchFailed {
name: name.to_string(),
url: baseurl.to_string(),
reason: String::from_utf8_lossy(&output.stderr).to_string(),
});
}
let mut cmd = Command::new("git");
cmd.arg("checkout").arg(version).current_dir(dest);
let output = cmd.output().await.map_err(|e| FetchError::FetchFailed {
name: name.to_string(),
url: baseurl.to_string(),
reason: e.to_string(),
})?;
if !output.status.success() {
return Err(FetchError::FetchFailed {
name: name.to_string(),
url: baseurl.to_string(),
reason: String::from_utf8_lossy(&output.stderr).to_string(),
});
}
Ok(())
}
/// Fetches a plugin from a git repository.
///
/// Parses the URL for version info, then clones and checks out the appropriate ref.
/// For Mixed version, tries the tag first and falls back to the commit hash.
pub async fn fetch_git_plugin(
path: &str,
name: &str,
basedir: &Path,
shallow: Option<bool>,
) -> Result<(), FetchError> {
let (baseurl, version) = parse_git_url(path);
match version {
GitVersion::None => {
log::error!("Version should be specified for git plugin URL: {}", path);
Err(FetchError::FetchFailed {
name: name.to_string(),
url: path.to_string(),
reason: "Version should be specified as either a tag or a commit hash.".to_string(),
})
}
GitVersion::Single(version) => {
log::debug!("Fetching git plugin: {}@{}", name, version);
let shallow_flag = shallow.unwrap_or(false);
git_clone_and_checkout(
&baseurl,
&version.to_string(),
&basedir.join(name),
name,
shallow_flag,
)
.await
}
GitVersion::Mixed(tag, commit) => {
log::debug!("Fetching git plugin: {}@{}/{}", name, tag, commit);
let shallow_flag = shallow.unwrap_or(false);
let result =
git_clone_and_checkout(&baseurl, &tag, &basedir.join(name), name, shallow_flag)
.await;
if result.is_err() {
log::warn!(
"Failed to clone with tag, trying commit hash. URL: {}",
path
);
return git_clone_and_checkout(&baseurl, &commit, &basedir.join(name), name, false)
.await;
}
result
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_git_url_no_version() {
let (url, version) = parse_git_url("https://github.com/user/repo");
assert_eq!(url, "https://github.com/user/repo");
assert_eq!(version, GitVersion::None);
}
#[test]
fn test_parse_git_url_single_tag() {
let (url, version) = parse_git_url("https://github.com/user/repo@v1.0.0");
assert_eq!(url, "https://github.com/user/repo");
assert_eq!(version, GitVersion::Single("v1.0.0".to_string()));
}
#[test]
fn test_parse_git_url_mixed_version() {
let (url, version) = parse_git_url("https://github.com/user/repo@main:abc123");
assert_eq!(url, "https://github.com/user/repo");
assert_eq!(
version,
GitVersion::Mixed("main".to_string(), "abc123".to_string())
);
}
#[test]
fn test_parse_git_url_complex_url() {
let (url, version) = parse_git_url("git@github.com:user/repo.git@tag:commit");
assert_eq!(url, "git@github.com:user/repo.git");
assert_eq!(
version,
GitVersion::Mixed("tag".to_string(), "commit".to_string())
);
}
#[test]
fn test_parse_git_url_single_with_at_in_path() {
let (url, version) = parse_git_url("https://github.com/user/repo@my-repo@v1.0");
assert_eq!(url, "https://github.com/user/repo@my-repo");
assert_eq!(version, GitVersion::Single("v1.0".to_string()));
}
}
workshop-baker/src/utils/fetch/http.rs
-110
View File
@@ -1,110 +0,0 @@
// Code in this module PARTIALLY OR FULLY utilized AI Coding Agent.
use super::checksum::{detect_checksum_type, verify_checksum};
use super::extract::{CompressionType, extract_tar};
use super::error::FetchError;
use std::path::Path;
pub async fn download_to_file(url: &str, dest: &Path, name: &str) -> Result<(), FetchError> {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(300))
.build()
.map_err(|e| FetchError::General(format!("Failed to build HTTP client: {}", e)))?;
let response = client
.get(url)
.send()
.await
.map_err(|e| FetchError::FetchFailed {
name: name.to_string(),
url: url.to_string(),
reason: format!("HTTP request failed: {}", e),
})?;
let status = response.status();
if !status.is_success() {
return Err(FetchError::FetchFailed {
name: name.to_string(),
url: url.to_string(),
reason: format!("HTTP error: {}", status),
});
}
let bytes = response
.bytes()
.await
.map_err(|e| FetchError::FetchFailed {
name: name.to_string(),
url: url.to_string(),
reason: format!("Failed to read response body: {}", e),
})?;
tokio::fs::write(dest, &bytes)
.await
.map_err(|e| FetchError::FetchFailed {
name: name.to_string(),
url: url.to_string(),
reason: format!("Failed to write file: {}", e),
})?;
Ok(())
}
fn extract_archive_extension(url: &str) -> &str {
let path_part = url
.split('?')
.next()
.unwrap_or(url)
.split('#')
.next()
.unwrap_or(url);
let file_part = path_part.rsplit('/').next().unwrap_or("");
let lower = file_part.to_lowercase();
if lower.ends_with(".tar.gz") || lower.ends_with(".tgz") {
".tar.gz"
} else if lower.ends_with(".tar.zst")
|| lower.ends_with(".tar.zstd")
|| lower.ends_with(".tzst")
{
".tar.zst"
} else if lower.ends_with(".tar") {
".tar"
} else {
".unknown"
}
}
pub async fn fetch_http_plugin(
url: &str,
name: &str,
basedir: &Path,
checksum_str: Option<&str>,
) -> Result<(), FetchError> {
let temp_dir = std::env::temp_dir().join("workshop-plugins");
tokio::fs::create_dir_all(&temp_dir).await.ok();
let ext = extract_archive_extension(url);
let temp_file = temp_dir.join(format!("{}-{}{}", name, uuid::Uuid::new_v4(), ext));
download_to_file(url, &temp_file, name).await?;
if let Some(checksum) = checksum_str
&& !checksum.is_empty()
{
let bytes = tokio::fs::read(&temp_file).await.map_err(|e| {
FetchError::General(format!("Failed to read downloaded file: {}", e))
})?;
let checksum_type = detect_checksum_type(checksum).map_err(FetchError::General)?;
if let Some(ct) = checksum_type {
verify_checksum(&bytes, checksum, ct).map_err(FetchError::General)?;
}
}
let compression = CompressionType::from_path(&temp_file).map_err(FetchError::General)?;
let destdir = basedir.join(name);
extract_tar(&temp_file, &destdir, compression).await?;
tokio::fs::remove_file(&temp_file).await.ok();
Ok(())
}
workshop-baker/src/utils/fetch/scheme.rs
-41
View File
@@ -1,41 +0,0 @@
use std::path::PathBuf;
use super::error::FetchError;
pub enum Scheme {
Git { repo: String, git_ref: String },
Http { url: String, checksum: Option<String> },
File { path: PathBuf },
}
/// Parse a URL into a Scheme.
/// Handles: git://, http://, https://, file://
/// workspace:// must be resolved by the caller to file:// before calling.
pub fn parse_url(url: &str) -> Result<Scheme, FetchError> {
let (scheme_str, rest) = url
.split_once("://")
.ok_or_else(|| FetchError::UnknownScheme(url.to_string()))?;
match scheme_str {
"git" => {
// Format: git://host/path@tag or git://host/path@tag:commit
let (base, version) = rest
.rsplit_once('@')
.map(|(b, v)| (b.to_string(), v.to_string()))
.unwrap_or_else(|| (rest.to_string(), String::new()));
Ok(Scheme::Git {
repo: format!("git://{}", base),
git_ref: version,
})
}
"http" | "https" => Ok(Scheme::Http {
url: url.to_string(),
checksum: None,
}),
"file" => {
let path = PathBuf::from(rest);
Ok(Scheme::File { path })
}
_ => Err(FetchError::UnknownScheme(scheme_str.to_string())),
}
}
workshop-baker/src/utils/fetch/types.rs
-84
View File
@@ -1,84 +0,0 @@
// Code in this module PARTIALLY OR FULLY utilized AI Coding Agent.
use serde::{Deserialize, Serialize};
/// Supported checksum types for verifying downloaded artifacts.
#[derive(Debug, Clone, PartialEq)]
pub enum ChecksumType {
/// SHA-256 hash (64 hex characters)
Sha256,
/// SHA-512 hash (128 hex characters)
Sha512,
}
/// Arguments for fetching a plugin artifact.
///
/// Contains optional checksum verification and shallow clone settings.
#[derive(Debug, Deserialize, Serialize, Clone, Default)]
pub struct FetchArgument {
/// SHA256 or SHA512 hex string for integrity verification.
#[serde(default)]
pub checksum: Option<String>,
/// Perform a shallow clone (single revision only).
#[serde(default)]
pub shallow: Option<bool>,
}
impl FetchArgument {
/// Returns the checksum if present and non-empty.
pub fn get_checksum(&self) -> Option<&str> {
self.checksum.as_deref().filter(|s| !s.is_empty())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_get_checksum_some_non_empty() {
let arg = FetchArgument {
checksum: Some("abc123".to_string()),
shallow: None,
};
assert_eq!(arg.get_checksum(), Some("abc123"));
}
#[test]
fn test_get_checksum_empty_string() {
let arg = FetchArgument {
checksum: Some("".to_string()),
shallow: None,
};
assert_eq!(arg.get_checksum(), None);
}
#[test]
fn test_get_checksum_none() {
let arg = FetchArgument {
checksum: None,
shallow: None,
};
assert_eq!(arg.get_checksum(), None);
}
#[test]
fn test_serde_roundtrip() {
let arg = FetchArgument {
checksum: Some("deadbeef".to_string()),
shallow: Some(true),
};
let json = serde_json::to_string(&arg).unwrap();
let parsed: FetchArgument = serde_json::from_str(&json).unwrap();
assert_eq!(parsed.checksum, arg.checksum);
assert_eq!(parsed.shallow, arg.shallow);
}
#[test]
fn test_serde_roundtrip_empty() {
let arg = FetchArgument::default();
let json = serde_json::to_string(&arg).unwrap();
let parsed: FetchArgument = serde_json::from_str(&json).unwrap();
assert_eq!(parsed.checksum, arg.checksum);
assert_eq!(parsed.shallow, arg.shallow);
}
}
workshop-baker/src/utils/fetch_plugin.rs
-73
View File
@@ -1,73 +0,0 @@
use std::collections::HashSet;
use std::path::Path;
use crate::finalize::RawPluginMap;
use crate::utils::fetch::{self, FetchError};
pub use crate::utils::fetch::FetchArgument;
/// Fetch a single plugin by URL.
///
/// Supports:
/// - `workspace://rel/path` → resolved to `file:///abs/path`
/// - `git://host/repo@tag` → git clone
/// - `https://url/archive.tar.gz` → HTTP download + checksum + extract
/// - `file:///path` → local file copy
pub async fn fetch_plugin(
url: &str,
name: &str,
basedir: &Path,
_argument: FetchArgument,
) -> Result<(), FetchError> {
let resolved_url = if let Some(rel) = url.strip_prefix("workspace://") {
format!(
"file://{}",
std::env::current_dir()
.unwrap_or_default()
.join(rel)
.display()
)
} else {
url.to_string()
};
fetch::fetch_resource(name, &resolved_url, basedir)
.await
.map(|_| ())
}
/// Fetch all listed plugins from a RawPluginMap.
///
/// Iterates over the plugin map, fetches each plugin that is in the
/// optional `list`, and updates `config.runtime.fspath` with the
/// downloaded path.
pub async fn fetch_plugins(
plugin_path: &Path,
plugin_config: &mut RawPluginMap,
list: Option<&HashSet<String>>,
) -> Result<(), FetchError> {
log::info!("Preparing to download {} plugins", plugin_config.len());
for (name, data) in plugin_config.iter_mut() {
if let Some(list) = list {
if !list.contains(name) {
log::debug!("Plugin {} not in list, skipping", name);
continue;
}
}
log::debug!("Downloading plugin {}", name);
let (source, config) = data.iter_mut().next().ok_or_else(|| {
FetchError::FetchFailed {
name: name.clone(),
url: String::new(),
reason: "no source configured".to_string(),
}
})?;
let fetch_argument = FetchArgument {
checksum: config.checksum.clone(),
shallow: config.shallow,
};
fetch_plugin(source, name, plugin_path, fetch_argument).await?;
log::debug!("Storing plugin {} to {}", name, plugin_path.display());
config.runtime.fspath = Some(plugin_path.join(name));
}
Ok(())
}
workshop-getterurl/Cargo.lock
Generated
+23
View File
@@ -581,6 +581,16 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "md-5"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"
dependencies = [
"cfg-if",
"digest",
]
[[package]]
name = "memchr"
version = "2.8.0"
@@ -970,6 +980,17 @@ dependencies = [
"syn",
]
[[package]]
name = "sha1"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "sha2"
version = "0.11.0"
@@ -1566,7 +1587,9 @@ name = "workshop-getterurl"
version = "0.1.0"
dependencies = [
"log",
"md-5",
"reqwest",
"sha1",
"sha2",
"thiserror",
"tokio",
workshop-getterurl/Cargo.toml
+7
View File
@@ -5,8 +5,15 @@ edition = "2024"
[dependencies]
log = "0.4.29"
md-5 = { version = "0.11.0", optional = true }
reqwest = { version = "0.13.3", default-features = false, features = ["rustls"] }
sha1 = { version = "0.11.0", optional = true }
sha2 = "0.11.0"
thiserror = "2.0.18"
tokio = { version = "1.52.3", features = ["process", "io-util", "fs", "rt-multi-thread", "macros"] }
url = "2.5.8"
[features]
default = ["md5", "sha1"]
md5 = ["dep:md-5"]
sha1 = ["dep:sha1"]
workshop-getterurl/src/builtin/mod.rs → workshop-getterurl/src/builtin.rs
View File
workshop-getterurl/src/builtin/http.rs
+58 -10
View File
@@ -1,4 +1,8 @@
use sha2::Digest;
#[cfg(feature = "md5")]
use md5::Md5;
#[cfg(feature = "sha1")]
use sha1::Sha1;
use std::future::Future;
use std::path::{Path, PathBuf};
use std::pin::Pin;
@@ -77,28 +81,42 @@ fn extract_filename(url: &Url) -> String {
.to_string()
}
/// Verify data against spec like "sha256:abc123". Prefix optional (guessed by length).
/// Verify data against spec like "sha256:abc123".
/// Prefix optional — guessed by length: MD5(32), SHA1(40), SHA256(64), SHA512(128).
/// MD5 and SHA1 require `--features md5,sha1` (enabled by default).
/// Other lengths produce an explicit error.
fn verify_checksum(data: &[u8], spec: &str) -> Result<(), GetterError> {
let (algo, expected) = if let Some((prefix, value)) = spec.split_once(':') {
(prefix, value)
} else {
let trimmed = spec.trim();
match trimmed.len() {
#[cfg(feature = "md5")]
32 => ("md5", trimmed),
#[cfg(not(feature = "md5"))]
32 => return Err(GetterError::General("MD5 not supported; enable `md5` feature".into())),
#[cfg(feature = "sha1")]
40 => ("sha1", trimmed),
#[cfg(not(feature = "sha1"))]
40 => return Err(GetterError::General("SHA1 not supported; enable `sha1` feature".into())),
64 => ("sha256", trimmed),
128 => ("sha512", trimmed),
_ => ("sha256", trimmed),
n => {
return Err(GetterError::General(format!(
"checksum length {} cannot be auto-detected; use prefix like sha256:, sha512:",
n
)));
}
}
};
let actual_hex: String = match algo {
"sha256" => sha2::Sha256::digest(data)
.iter()
.map(|b| format!("{:02x}", b))
.collect(),
"sha512" => sha2::Sha512::digest(data)
.iter()
.map(|b| format!("{:02x}", b))
.collect(),
#[cfg(feature = "md5")]
"md5" => Md5::digest(data).iter().map(|b| format!("{:02x}", b)).collect(),
#[cfg(feature = "sha1")]
"sha1" => Sha1::digest(data).iter().map(|b| format!("{:02x}", b)).collect(),
"sha256" => sha2::Sha256::digest(data).iter().map(|b| format!("{:02x}", b)).collect(),
"sha512" => sha2::Sha512::digest(data).iter().map(|b| format!("{:02x}", b)).collect(),
_ => return Err(GetterError::General(format!("Unknown algorithm: {}", algo))),
};
@@ -142,4 +160,34 @@ mod tests {
let spec = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
verify_checksum(data, spec).unwrap();
}
#[cfg(feature = "md5")]
#[test]
fn test_verify_md5() {
let data = b"hello";
let spec = "md5:5d41402abc4b2a76b9719d911017c592";
verify_checksum(data, spec).unwrap();
}
#[cfg(feature = "sha1")]
#[test]
fn test_verify_sha1() {
let data = b"hello";
let spec = "sha1:aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d";
verify_checksum(data, spec).unwrap();
}
#[cfg(feature = "md5")]
#[test]
fn test_verify_md5_no_prefix() {
let data = b"hello";
let spec = "5d41402abc4b2a76b9719d911017c592";
verify_checksum(data, spec).unwrap();
}
#[test]
fn test_verify_unknown_length_errors() {
let spec = "0".repeat(50);
assert!(verify_checksum(b"data", &spec).is_err());
}
}