vault: implement bootstrap integration
This commit is contained in:
@@ -9,9 +9,10 @@ from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
|
||||
from generators import generate_host_files
|
||||
from manipulators import update_flake_nix, update_terraform_vms
|
||||
from generators import generate_host_files, generate_vault_terraform
|
||||
from manipulators import update_flake_nix, update_terraform_vms, add_wrapped_token_to_vm
|
||||
from models import HostConfig
|
||||
from vault_helper import generate_wrapped_token
|
||||
from validators import (
|
||||
validate_hostname_format,
|
||||
validate_hostname_unique,
|
||||
@@ -46,6 +47,8 @@ def main(
|
||||
disk: str = typer.Option("20G", "--disk", help="Disk size (e.g., 20G, 50G, 100G)"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without creating files"),
|
||||
force: bool = typer.Option(False, "--force", help="Overwrite existing host configuration"),
|
||||
skip_vault: bool = typer.Option(False, "--skip-vault", help="Skip Vault configuration and token generation"),
|
||||
regenerate_token: bool = typer.Option(False, "--regenerate-token", help="Only regenerate Vault wrapped token (no other changes)"),
|
||||
) -> None:
|
||||
"""
|
||||
Create a new NixOS host configuration.
|
||||
@@ -58,6 +61,51 @@ def main(
|
||||
ctx.get_help()
|
||||
sys.exit(1)
|
||||
|
||||
# Get repository root
|
||||
repo_root = get_repo_root()
|
||||
|
||||
# Handle token regeneration mode
|
||||
if regenerate_token:
|
||||
# Validate that incompatible options aren't used
|
||||
if force or dry_run or skip_vault:
|
||||
console.print("[bold red]Error:[/bold red] --regenerate-token cannot be used with --force, --dry-run, or --skip-vault\n")
|
||||
sys.exit(1)
|
||||
if ip or cpu != 2 or memory != 2048 or disk != "20G":
|
||||
console.print("[bold red]Error:[/bold red] --regenerate-token only regenerates the token. Other options (--ip, --cpu, --memory, --disk) are ignored.\n")
|
||||
console.print("[yellow]Tip:[/yellow] Use without those options, or use --force to update the entire configuration.\n")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
console.print(f"\n[bold blue]Regenerating Vault token for {hostname}...[/bold blue]")
|
||||
|
||||
# Validate hostname exists
|
||||
host_dir = repo_root / "hosts" / hostname
|
||||
if not host_dir.exists():
|
||||
console.print(f"[bold red]Error:[/bold red] Host {hostname} does not exist")
|
||||
console.print(f"Host directory not found: {host_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
# Generate new wrapped token
|
||||
wrapped_token = generate_wrapped_token(hostname, repo_root)
|
||||
|
||||
# Update only the wrapped token in vms.tf
|
||||
add_wrapped_token_to_vm(hostname, wrapped_token, repo_root)
|
||||
console.print("[green]✓[/green] Regenerated and updated wrapped token in terraform/vms.tf")
|
||||
|
||||
console.print("\n[bold green]✓ Token regenerated successfully![/bold green]")
|
||||
console.print(f"\n[yellow]⚠️[/yellow] Token expires in 24 hours")
|
||||
console.print(f"[yellow]⚠️[/yellow] Deploy the VM within 24h or regenerate token again\n")
|
||||
|
||||
console.print("[bold cyan]Next steps:[/bold cyan]")
|
||||
console.print(f" cd terraform && tofu apply")
|
||||
console.print(f" # Then redeploy VM to pick up new token\n")
|
||||
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"\n[bold red]Error regenerating token:[/bold red] {e}\n")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
# Build configuration
|
||||
config = HostConfig(
|
||||
@@ -68,9 +116,6 @@ def main(
|
||||
disk=disk,
|
||||
)
|
||||
|
||||
# Get repository root
|
||||
repo_root = get_repo_root()
|
||||
|
||||
# Validate configuration
|
||||
console.print("\n[bold blue]Validating configuration...[/bold blue]")
|
||||
|
||||
@@ -116,11 +161,34 @@ def main(
|
||||
update_terraform_vms(config, repo_root, force=force)
|
||||
console.print("[green]✓[/green] Updated terraform/vms.tf")
|
||||
|
||||
# Generate Vault configuration if not skipped
|
||||
if not skip_vault:
|
||||
console.print("\n[bold blue]Configuring Vault integration...[/bold blue]")
|
||||
|
||||
try:
|
||||
# Generate Vault Terraform configuration
|
||||
generate_vault_terraform(hostname, repo_root)
|
||||
console.print("[green]✓[/green] Updated terraform/vault/hosts-generated.tf")
|
||||
|
||||
# Generate wrapped token
|
||||
wrapped_token = generate_wrapped_token(hostname, repo_root)
|
||||
|
||||
# Add wrapped token to VM configuration
|
||||
add_wrapped_token_to_vm(hostname, wrapped_token, repo_root)
|
||||
console.print("[green]✓[/green] Added wrapped token to terraform/vms.tf")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"\n[yellow]⚠️ Vault configuration failed: {e}[/yellow]")
|
||||
console.print("[yellow]Host configuration created without Vault integration[/yellow]")
|
||||
console.print("[yellow]You can add Vault support later by re-running with --force[/yellow]\n")
|
||||
else:
|
||||
console.print("\n[yellow]Skipped Vault configuration (--skip-vault)[/yellow]")
|
||||
|
||||
# Success message
|
||||
console.print("\n[bold green]✓ Host configuration generated successfully![/bold green]\n")
|
||||
|
||||
# Display next steps
|
||||
display_next_steps(hostname)
|
||||
display_next_steps(hostname, skip_vault=skip_vault)
|
||||
|
||||
except ValueError as e:
|
||||
console.print(f"\n[bold red]Error:[/bold red] {e}\n", style="red")
|
||||
@@ -164,8 +232,18 @@ def display_dry_run_summary(config: HostConfig, repo_root: Path) -> None:
|
||||
console.print(f" • {repo_root}/terraform/vms.tf (add VM definition)")
|
||||
|
||||
|
||||
def display_next_steps(hostname: str) -> None:
|
||||
def display_next_steps(hostname: str, skip_vault: bool = False) -> None:
|
||||
"""Display next steps after successful generation."""
|
||||
vault_files = "" if skip_vault else " terraform/vault/hosts-generated.tf"
|
||||
vault_apply = ""
|
||||
|
||||
if not skip_vault:
|
||||
vault_apply = """
|
||||
4a. Apply Vault configuration:
|
||||
[white]cd terraform/vault
|
||||
tofu apply[/white]
|
||||
"""
|
||||
|
||||
next_steps = f"""[bold cyan]Next Steps:[/bold cyan]
|
||||
|
||||
1. Review changes:
|
||||
@@ -181,14 +259,16 @@ def display_next_steps(hostname: str) -> None:
|
||||
tofu plan[/white]
|
||||
|
||||
4. Commit changes:
|
||||
[white]git add hosts/{hostname} flake.nix terraform/vms.tf
|
||||
[white]git add hosts/{hostname} flake.nix terraform/vms.tf{vault_files}
|
||||
git commit -m "hosts: add {hostname} configuration"[/white]
|
||||
|
||||
5. Deploy VM (after merging to master):
|
||||
{vault_apply}
|
||||
5. Deploy VM (after merging to master or within 24h of token generation):
|
||||
[white]cd terraform
|
||||
tofu apply[/white]
|
||||
|
||||
6. Bootstrap the host (see Phase 3 of deployment pipeline)
|
||||
6. Host will bootstrap automatically on first boot
|
||||
- Wrapped token expires in 24 hours
|
||||
- If expired, re-run: create-host --hostname {hostname} --force
|
||||
"""
|
||||
console.print(Panel(next_steps, border_style="cyan"))
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ python3Packages.buildPythonApplication {
|
||||
typer
|
||||
jinja2
|
||||
rich
|
||||
hvac # Python Vault/OpenBao client library
|
||||
];
|
||||
|
||||
# Install templates to share directory
|
||||
|
||||
@@ -86,3 +86,114 @@ def generate_host_files(config: HostConfig, repo_root: Path) -> None:
|
||||
state_version=config.state_version,
|
||||
)
|
||||
(host_dir / "configuration.nix").write_text(config_content)
|
||||
|
||||
|
||||
def generate_vault_terraform(hostname: str, repo_root: Path) -> None:
|
||||
"""
|
||||
Generate or update Vault Terraform configuration for a new host.
|
||||
|
||||
Creates/updates terraform/vault/hosts-generated.tf with:
|
||||
- Host policy granting access to hosts/<hostname>/* secrets
|
||||
- AppRole configuration for the host
|
||||
- Placeholder secret entry (user adds actual secrets separately)
|
||||
|
||||
Args:
|
||||
hostname: Hostname for the new host
|
||||
repo_root: Path to repository root
|
||||
"""
|
||||
vault_tf_path = repo_root / "terraform" / "vault" / "hosts-generated.tf"
|
||||
|
||||
# Read existing file if it exists, otherwise start with empty structure
|
||||
if vault_tf_path.exists():
|
||||
content = vault_tf_path.read_text()
|
||||
else:
|
||||
# Create initial file structure
|
||||
content = """# WARNING: Auto-generated by create-host tool
|
||||
# Manual edits will be overwritten when create-host is run
|
||||
|
||||
# Generated host policies
|
||||
# Each host gets access to its own secrets under hosts/<hostname>/*
|
||||
locals {
|
||||
generated_host_policies = {
|
||||
}
|
||||
|
||||
# Placeholder secrets - user should add actual secrets manually or via tofu
|
||||
generated_secrets = {
|
||||
}
|
||||
}
|
||||
|
||||
# Create policies for generated hosts
|
||||
resource "vault_policy" "generated_host_policies" {
|
||||
for_each = local.generated_host_policies
|
||||
|
||||
name = "host-\${each.key}"
|
||||
|
||||
policy = <<-EOT
|
||||
# Allow host to read its own secrets
|
||||
%{for path in each.value.paths~}
|
||||
path "${path}" {
|
||||
capabilities = ["read", "list"]
|
||||
}
|
||||
%{endfor~}
|
||||
EOT
|
||||
}
|
||||
|
||||
# Create AppRoles for generated hosts
|
||||
resource "vault_approle_auth_backend_role" "generated_hosts" {
|
||||
for_each = local.generated_host_policies
|
||||
|
||||
backend = vault_auth_backend.approle.path
|
||||
role_name = each.key
|
||||
token_policies = ["host-\${each.key}"]
|
||||
secret_id_ttl = 0 # Never expire (wrapped tokens provide time limit)
|
||||
token_ttl = 3600
|
||||
token_max_ttl = 3600
|
||||
secret_id_num_uses = 0 # Unlimited uses
|
||||
}
|
||||
"""
|
||||
|
||||
# Parse existing policies from the file
|
||||
import re
|
||||
|
||||
policies_match = re.search(
|
||||
r'generated_host_policies = \{(.*?)\n \}',
|
||||
content,
|
||||
re.DOTALL
|
||||
)
|
||||
|
||||
if policies_match:
|
||||
policies_content = policies_match.group(1)
|
||||
else:
|
||||
policies_content = ""
|
||||
|
||||
# Check if hostname already exists
|
||||
if f'"{hostname}"' in policies_content:
|
||||
# Already exists, don't duplicate
|
||||
return
|
||||
|
||||
# Add new policy entry
|
||||
new_policy = f'''
|
||||
"{hostname}" = {{
|
||||
paths = [
|
||||
"secret/data/hosts/{hostname}/*",
|
||||
]
|
||||
}}'''
|
||||
|
||||
# Insert before the closing brace
|
||||
if policies_content.strip():
|
||||
# There are existing entries, add after them
|
||||
new_policies_content = policies_content.rstrip() + new_policy + "\n "
|
||||
else:
|
||||
# First entry
|
||||
new_policies_content = new_policy + "\n "
|
||||
|
||||
# Replace the policies map
|
||||
new_content = re.sub(
|
||||
r'(generated_host_policies = \{)(.*?)(\n \})',
|
||||
rf'\1{new_policies_content}\3',
|
||||
content,
|
||||
flags=re.DOTALL
|
||||
)
|
||||
|
||||
# Write the updated file
|
||||
vault_tf_path.write_text(new_content)
|
||||
|
||||
@@ -122,3 +122,63 @@ def update_terraform_vms(config: HostConfig, repo_root: Path, force: bool = Fals
|
||||
)
|
||||
|
||||
terraform_path.write_text(new_content)
|
||||
|
||||
|
||||
def add_wrapped_token_to_vm(hostname: str, wrapped_token: str, repo_root: Path) -> None:
|
||||
"""
|
||||
Add or update the vault_wrapped_token field in an existing VM entry.
|
||||
|
||||
Args:
|
||||
hostname: Hostname of the VM
|
||||
wrapped_token: The wrapped token to add
|
||||
repo_root: Path to repository root
|
||||
"""
|
||||
terraform_path = repo_root / "terraform" / "vms.tf"
|
||||
content = terraform_path.read_text()
|
||||
|
||||
# Find the VM entry
|
||||
hostname_pattern = rf'^\s+"{re.escape(hostname)}" = \{{'
|
||||
match = re.search(hostname_pattern, content, re.MULTILINE)
|
||||
|
||||
if not match:
|
||||
raise ValueError(f"Could not find VM entry for {hostname} in terraform/vms.tf")
|
||||
|
||||
# Find the full VM block
|
||||
block_pattern = rf'(^\s+"{re.escape(hostname)}" = \{{)(.*?)(^\s+\}})'
|
||||
block_match = re.search(block_pattern, content, re.MULTILINE | re.DOTALL)
|
||||
|
||||
if not block_match:
|
||||
raise ValueError(f"Could not parse VM block for {hostname}")
|
||||
|
||||
block_start = block_match.group(1)
|
||||
block_content = block_match.group(2)
|
||||
block_end = block_match.group(3)
|
||||
|
||||
# Check if vault_wrapped_token already exists
|
||||
if "vault_wrapped_token" in block_content:
|
||||
# Update existing token
|
||||
block_content = re.sub(
|
||||
r'vault_wrapped_token\s*=\s*"[^"]*"',
|
||||
f'vault_wrapped_token = "{wrapped_token}"',
|
||||
block_content
|
||||
)
|
||||
else:
|
||||
# Add new token field (add before closing brace)
|
||||
# Find the last field and add after it
|
||||
block_content = block_content.rstrip()
|
||||
if block_content and not block_content.endswith("\n"):
|
||||
block_content += "\n"
|
||||
block_content += f' vault_wrapped_token = "{wrapped_token}"\n'
|
||||
|
||||
# Reconstruct the block
|
||||
new_block = block_start + block_content + block_end
|
||||
|
||||
# Replace in content
|
||||
new_content = re.sub(
|
||||
rf'^\s+"{re.escape(hostname)}" = \{{.*?^\s+\}}',
|
||||
new_block,
|
||||
content,
|
||||
flags=re.MULTILINE | re.DOTALL
|
||||
)
|
||||
|
||||
terraform_path.write_text(new_content)
|
||||
|
||||
@@ -14,6 +14,7 @@ setup(
|
||||
"validators",
|
||||
"generators",
|
||||
"manipulators",
|
||||
"vault_helper",
|
||||
],
|
||||
include_package_data=True,
|
||||
data_files=[
|
||||
@@ -23,6 +24,7 @@ setup(
|
||||
"typer",
|
||||
"jinja2",
|
||||
"rich",
|
||||
"hvac",
|
||||
],
|
||||
entry_points={
|
||||
"console_scripts": [
|
||||
|
||||
178
scripts/create-host/vault_helper.py
Normal file
178
scripts/create-host/vault_helper.py
Normal file
@@ -0,0 +1,178 @@
|
||||
"""Helper functions for Vault/OpenBao API interactions."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import hvac
|
||||
import typer
|
||||
|
||||
|
||||
def get_vault_client(vault_addr: Optional[str] = None, vault_token: Optional[str] = None) -> hvac.Client:
|
||||
"""
|
||||
Get a Vault client instance.
|
||||
|
||||
Args:
|
||||
vault_addr: Vault server address (defaults to BAO_ADDR env var or hardcoded default)
|
||||
vault_token: Vault token (defaults to BAO_TOKEN env var or prompts user)
|
||||
|
||||
Returns:
|
||||
Configured hvac.Client instance
|
||||
|
||||
Raises:
|
||||
typer.Exit: If unable to create client or authenticate
|
||||
"""
|
||||
# Get Vault address
|
||||
if vault_addr is None:
|
||||
vault_addr = os.getenv("BAO_ADDR", "https://vault01.home.2rjus.net:8200")
|
||||
|
||||
# Get Vault token
|
||||
if vault_token is None:
|
||||
vault_token = os.getenv("BAO_TOKEN")
|
||||
|
||||
if not vault_token:
|
||||
typer.echo("\n⚠️ Vault token required. Set BAO_TOKEN environment variable or enter it below.")
|
||||
vault_token = typer.prompt("Vault token (BAO_TOKEN)", hide_input=True)
|
||||
|
||||
# Create client
|
||||
try:
|
||||
client = hvac.Client(url=vault_addr, token=vault_token, verify=False)
|
||||
|
||||
# Verify authentication
|
||||
if not client.is_authenticated():
|
||||
typer.echo(f"\n❌ Failed to authenticate to Vault at {vault_addr}", err=True)
|
||||
typer.echo("Check your BAO_TOKEN and ensure Vault is accessible", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
return client
|
||||
|
||||
except Exception as e:
|
||||
typer.echo(f"\n❌ Error connecting to Vault: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
def generate_wrapped_token(hostname: str, repo_root: Path) -> str:
|
||||
"""
|
||||
Generate a wrapped token containing AppRole credentials for a host.
|
||||
|
||||
This function:
|
||||
1. Applies Terraform to ensure the AppRole exists
|
||||
2. Reads the role_id for the host
|
||||
3. Generates a secret_id
|
||||
4. Wraps both credentials in a cubbyhole token (24h TTL, single-use)
|
||||
|
||||
Args:
|
||||
hostname: The host to generate credentials for
|
||||
repo_root: Path to repository root (for running terraform)
|
||||
|
||||
Returns:
|
||||
Wrapped token string (hvs.CAES...)
|
||||
|
||||
Raises:
|
||||
typer.Exit: If Terraform fails or Vault operations fail
|
||||
"""
|
||||
from rich.console import Console
|
||||
|
||||
console = Console()
|
||||
|
||||
# Get Vault client
|
||||
client = get_vault_client()
|
||||
|
||||
# First, apply Terraform to ensure AppRole exists
|
||||
console.print(f"\n[bold blue]Applying Vault Terraform configuration...[/bold blue]")
|
||||
terraform_dir = repo_root / "terraform" / "vault"
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["tofu", "apply", "-auto-approve"],
|
||||
cwd=terraform_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
console.print(f"[red]❌ Terraform apply failed:[/red]")
|
||||
console.print(result.stderr)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
console.print("[green]✓[/green] Terraform applied successfully")
|
||||
|
||||
except FileNotFoundError:
|
||||
console.print(f"[red]❌ Error: 'tofu' command not found[/red]")
|
||||
console.print("Ensure OpenTofu is installed and in PATH")
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
# Read role_id
|
||||
try:
|
||||
console.print(f"[bold blue]Reading AppRole credentials for {hostname}...[/bold blue]")
|
||||
role_id_response = client.read(f"auth/approle/role/{hostname}/role-id")
|
||||
role_id = role_id_response["data"]["role_id"]
|
||||
console.print(f"[green]✓[/green] Retrieved role_id")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]❌ Failed to read role_id for {hostname}:[/red] {e}")
|
||||
console.print(f"\nEnsure the AppRole '{hostname}' exists in Vault")
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
# Generate secret_id
|
||||
try:
|
||||
secret_id_response = client.write(f"auth/approle/role/{hostname}/secret-id")
|
||||
secret_id = secret_id_response["data"]["secret_id"]
|
||||
console.print(f"[green]✓[/green] Generated secret_id")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]❌ Failed to generate secret_id:[/red] {e}")
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
# Wrap the credentials in a cubbyhole token
|
||||
try:
|
||||
console.print(f"[bold blue]Creating wrapped token (24h TTL, single-use)...[/bold blue]")
|
||||
|
||||
# Use the response wrapping feature to wrap our credentials
|
||||
# This creates a temporary token that can only be used once to retrieve the actual credentials
|
||||
wrap_response = client.write(
|
||||
"sys/wrapping/wrap",
|
||||
wrap_ttl="24h",
|
||||
# The data we're wrapping
|
||||
role_id=role_id,
|
||||
secret_id=secret_id,
|
||||
)
|
||||
|
||||
wrapped_token = wrap_response["wrap_info"]["token"]
|
||||
console.print(f"[green]✓[/green] Created wrapped token: {wrapped_token[:20]}...")
|
||||
console.print(f"[yellow]⚠️[/yellow] Token expires in 24 hours")
|
||||
console.print(f"[yellow]⚠️[/yellow] Token can only be used once")
|
||||
|
||||
return wrapped_token
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]❌ Failed to create wrapped token:[/red] {e}")
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
def verify_vault_setup(hostname: str) -> bool:
|
||||
"""
|
||||
Verify that Vault is properly configured for a host.
|
||||
|
||||
Checks:
|
||||
- Vault is accessible
|
||||
- AppRole exists for the hostname
|
||||
- Can read role_id
|
||||
|
||||
Args:
|
||||
hostname: The host to verify
|
||||
|
||||
Returns:
|
||||
True if everything is configured correctly, False otherwise
|
||||
"""
|
||||
try:
|
||||
client = get_vault_client()
|
||||
|
||||
# Try to read the role_id
|
||||
client.read(f"auth/approle/role/{hostname}/role-id")
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
return False
|
||||
78
scripts/vault-fetch/README.md
Normal file
78
scripts/vault-fetch/README.md
Normal file
@@ -0,0 +1,78 @@
|
||||
# vault-fetch
|
||||
|
||||
A helper script for fetching secrets from OpenBao/Vault and writing them to the filesystem.
|
||||
|
||||
## Features
|
||||
|
||||
- **AppRole Authentication**: Uses role_id and secret_id from `/var/lib/vault/approle/`
|
||||
- **Individual Secret Files**: Writes each secret key as a separate file for easy consumption
|
||||
- **Caching**: Maintains a cache of secrets for fallback when Vault is unreachable
|
||||
- **Graceful Degradation**: Falls back to cached secrets if Vault authentication fails
|
||||
- **Secure Permissions**: Sets 600 permissions on all secret files
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
vault-fetch <secret-path> <output-directory> [cache-directory]
|
||||
```
|
||||
|
||||
### Examples
|
||||
|
||||
```bash
|
||||
# Fetch Grafana admin secrets
|
||||
vault-fetch hosts/monitoring01/grafana-admin /run/secrets/grafana /var/lib/vault/cache/grafana
|
||||
|
||||
# Use default cache location
|
||||
vault-fetch hosts/monitoring01/grafana-admin /run/secrets/grafana
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Read Credentials**: Loads `role_id` and `secret_id` from `/var/lib/vault/approle/`
|
||||
2. **Authenticate**: Calls `POST /v1/auth/approle/login` to get a Vault token
|
||||
3. **Fetch Secret**: Retrieves secret from `GET /v1/secret/data/{path}`
|
||||
4. **Extract Keys**: Parses JSON response and extracts individual secret keys
|
||||
5. **Write Files**: Creates one file per secret key in output directory
|
||||
6. **Update Cache**: Copies secrets to cache directory for fallback
|
||||
7. **Set Permissions**: Ensures all files have 600 permissions (owner read/write only)
|
||||
|
||||
## Error Handling
|
||||
|
||||
If Vault is unreachable or authentication fails:
|
||||
- Script logs a warning to stderr
|
||||
- Falls back to cached secrets from previous successful fetch
|
||||
- Exits with error code 1 if no cache is available
|
||||
|
||||
## Environment Variables
|
||||
|
||||
- `VAULT_ADDR`: Vault server address (default: `https://vault01.home.2rjus.net:8200`)
|
||||
- `VAULT_SKIP_VERIFY`: Skip TLS verification (default: `1`)
|
||||
|
||||
## Integration with NixOS
|
||||
|
||||
This tool is designed to be called from systemd service `ExecStartPre` hooks via the `vault.secrets` NixOS module:
|
||||
|
||||
```nix
|
||||
vault.secrets.grafana-admin = {
|
||||
secretPath = "hosts/monitoring01/grafana-admin";
|
||||
};
|
||||
|
||||
# Service automatically gets secrets fetched before start
|
||||
systemd.services.grafana.serviceConfig = {
|
||||
EnvironmentFile = "/run/secrets/grafana-admin/password";
|
||||
};
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
- `curl`: For Vault API calls
|
||||
- `jq`: For JSON parsing
|
||||
- `coreutils`: For file operations
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- AppRole credentials stored at `/var/lib/vault/approle/` should be root-owned with 600 permissions
|
||||
- Tokens are ephemeral and not stored - fresh authentication on each fetch
|
||||
- Secrets written to tmpfs (`/run/secrets/`) are lost on reboot
|
||||
- Cache directory persists across reboots for service availability
|
||||
- All secret files have restrictive permissions (600)
|
||||
18
scripts/vault-fetch/default.nix
Normal file
18
scripts/vault-fetch/default.nix
Normal file
@@ -0,0 +1,18 @@
|
||||
{ pkgs, lib, ... }:
|
||||
|
||||
pkgs.writeShellApplication {
|
||||
name = "vault-fetch";
|
||||
|
||||
runtimeInputs = with pkgs; [
|
||||
curl # Vault API calls
|
||||
jq # JSON parsing
|
||||
coreutils # File operations
|
||||
];
|
||||
|
||||
text = builtins.readFile ./vault-fetch.sh;
|
||||
|
||||
meta = with lib; {
|
||||
description = "Fetch secrets from OpenBao/Vault and write to filesystem";
|
||||
license = licenses.mit;
|
||||
};
|
||||
}
|
||||
152
scripts/vault-fetch/vault-fetch.sh
Normal file
152
scripts/vault-fetch/vault-fetch.sh
Normal file
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# vault-fetch: Fetch secrets from OpenBao/Vault and write to filesystem
|
||||
#
|
||||
# Usage: vault-fetch <secret-path> <output-directory> [cache-directory]
|
||||
#
|
||||
# Example: vault-fetch hosts/monitoring01/grafana-admin /run/secrets/grafana /var/lib/vault/cache/grafana
|
||||
#
|
||||
# This script:
|
||||
# 1. Authenticates to Vault using AppRole credentials from /var/lib/vault/approle/
|
||||
# 2. Fetches secrets from the specified path
|
||||
# 3. Writes each secret key as an individual file in the output directory
|
||||
# 4. Updates cache for fallback when Vault is unreachable
|
||||
# 5. Falls back to cache if Vault authentication fails or is unreachable
|
||||
|
||||
# Parse arguments
|
||||
if [ $# -lt 2 ]; then
|
||||
echo "Usage: vault-fetch <secret-path> <output-directory> [cache-directory]" >&2
|
||||
echo "Example: vault-fetch hosts/monitoring01/grafana /run/secrets/grafana /var/lib/vault/cache/grafana" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SECRET_PATH="$1"
|
||||
OUTPUT_DIR="$2"
|
||||
CACHE_DIR="${3:-/var/lib/vault/cache/$(basename "$OUTPUT_DIR")}"
|
||||
|
||||
# Vault configuration
|
||||
VAULT_ADDR="${VAULT_ADDR:-https://vault01.home.2rjus.net:8200}"
|
||||
VAULT_SKIP_VERIFY="${VAULT_SKIP_VERIFY:-1}"
|
||||
APPROLE_DIR="/var/lib/vault/approle"
|
||||
|
||||
# TLS verification flag for curl
|
||||
if [ "$VAULT_SKIP_VERIFY" = "1" ]; then
|
||||
CURL_TLS_FLAG="-k"
|
||||
else
|
||||
CURL_TLS_FLAG=""
|
||||
fi
|
||||
|
||||
# Logging helper
|
||||
log() {
|
||||
echo "[vault-fetch] $*" >&2
|
||||
}
|
||||
|
||||
# Error handler
|
||||
error() {
|
||||
log "ERROR: $*"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check if cache is available
|
||||
has_cache() {
|
||||
[ -d "$CACHE_DIR" ] && [ -n "$(ls -A "$CACHE_DIR" 2>/dev/null)" ]
|
||||
}
|
||||
|
||||
# Use cached secrets
|
||||
use_cache() {
|
||||
if ! has_cache; then
|
||||
error "No cache available and Vault is unreachable"
|
||||
fi
|
||||
|
||||
log "WARNING: Using cached secrets from $CACHE_DIR"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
cp -r "$CACHE_DIR"/* "$OUTPUT_DIR/"
|
||||
chmod -R u=rw,go= "$OUTPUT_DIR"/*
|
||||
}
|
||||
|
||||
# Fetch secrets from Vault
|
||||
fetch_from_vault() {
|
||||
# Read AppRole credentials
|
||||
if [ ! -f "$APPROLE_DIR/role-id" ] || [ ! -f "$APPROLE_DIR/secret-id" ]; then
|
||||
log "WARNING: AppRole credentials not found at $APPROLE_DIR"
|
||||
use_cache
|
||||
return
|
||||
fi
|
||||
|
||||
ROLE_ID=$(cat "$APPROLE_DIR/role-id")
|
||||
SECRET_ID=$(cat "$APPROLE_DIR/secret-id")
|
||||
|
||||
# Authenticate to Vault
|
||||
log "Authenticating to Vault at $VAULT_ADDR"
|
||||
AUTH_RESPONSE=$(curl -s $CURL_TLS_FLAG -X POST \
|
||||
-d "{\"role_id\":\"$ROLE_ID\",\"secret_id\":\"$SECRET_ID\"}" \
|
||||
"$VAULT_ADDR/v1/auth/approle/login" 2>&1) || {
|
||||
log "WARNING: Failed to connect to Vault"
|
||||
use_cache
|
||||
return
|
||||
}
|
||||
|
||||
# Check for errors in response
|
||||
if echo "$AUTH_RESPONSE" | jq -e '.errors' >/dev/null 2>&1; then
|
||||
ERRORS=$(echo "$AUTH_RESPONSE" | jq -r '.errors[]' 2>/dev/null || echo "Unknown error")
|
||||
log "WARNING: Vault authentication failed: $ERRORS"
|
||||
use_cache
|
||||
return
|
||||
fi
|
||||
|
||||
# Extract token
|
||||
VAULT_TOKEN=$(echo "$AUTH_RESPONSE" | jq -r '.auth.client_token' 2>/dev/null)
|
||||
if [ -z "$VAULT_TOKEN" ] || [ "$VAULT_TOKEN" = "null" ]; then
|
||||
log "WARNING: Failed to extract Vault token from response"
|
||||
use_cache
|
||||
return
|
||||
fi
|
||||
|
||||
log "Successfully authenticated to Vault"
|
||||
|
||||
# Fetch secret
|
||||
log "Fetching secret from path: $SECRET_PATH"
|
||||
SECRET_RESPONSE=$(curl -s $CURL_TLS_FLAG \
|
||||
-H "X-Vault-Token: $VAULT_TOKEN" \
|
||||
"$VAULT_ADDR/v1/secret/data/$SECRET_PATH" 2>&1) || {
|
||||
log "WARNING: Failed to fetch secret from Vault"
|
||||
use_cache
|
||||
return
|
||||
}
|
||||
|
||||
# Check for errors
|
||||
if echo "$SECRET_RESPONSE" | jq -e '.errors' >/dev/null 2>&1; then
|
||||
ERRORS=$(echo "$SECRET_RESPONSE" | jq -r '.errors[]' 2>/dev/null || echo "Unknown error")
|
||||
log "WARNING: Failed to fetch secret: $ERRORS"
|
||||
use_cache
|
||||
return
|
||||
fi
|
||||
|
||||
# Extract secret data
|
||||
SECRET_DATA=$(echo "$SECRET_RESPONSE" | jq -r '.data.data' 2>/dev/null)
|
||||
if [ -z "$SECRET_DATA" ] || [ "$SECRET_DATA" = "null" ]; then
|
||||
log "WARNING: No secret data found at path $SECRET_PATH"
|
||||
use_cache
|
||||
return
|
||||
fi
|
||||
|
||||
# Create output and cache directories
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
mkdir -p "$CACHE_DIR"
|
||||
|
||||
# Write each secret key to a separate file
|
||||
log "Writing secrets to $OUTPUT_DIR"
|
||||
echo "$SECRET_DATA" | jq -r 'to_entries[] | "\(.key)\n\(.value)"' | while read -r key; read -r value; do
|
||||
echo -n "$value" > "$OUTPUT_DIR/$key"
|
||||
echo -n "$value" > "$CACHE_DIR/$key"
|
||||
chmod 600 "$OUTPUT_DIR/$key"
|
||||
chmod 600 "$CACHE_DIR/$key"
|
||||
log " - Wrote secret key: $key"
|
||||
done
|
||||
|
||||
log "Successfully fetched and cached secrets"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
fetch_from_vault
|
||||
Reference in New Issue
Block a user