Vendor Codex and land Phase 1 Epiphany state slice

This commit is contained in:
Meta Vens 2026-04-23 23:20:16 +01:00
commit 2042687e30
3959 changed files with 953751 additions and 7 deletions

View file

@ -0,0 +1,23 @@
# npm releases
Use the staging helper in the repo root to generate npm tarballs for a release. For
example, to stage the CLI, responses proxy, and SDK packages for version `0.6.0`:
```bash
./scripts/stage_npm_packages.py \
--release-version 0.6.0 \
--package codex \
--package codex-responses-api-proxy \
--package codex-sdk
```
This downloads the native artifacts once, hydrates `vendor/` for each package, and writes
tarballs to `dist/npm/`.
When `--package codex` is provided, the staging helper builds the lightweight
`@openai/codex` meta package plus all platform-native `@openai/codex` variants
that are later published under platform-specific dist-tags.
If you need to invoke `build_npm_package.py` directly, run
`codex-cli/scripts/install_native_deps.py` first and pass `--vendor-src` pointing to the
directory that contains the populated `vendor/` tree.

View file

@ -0,0 +1,16 @@
#!/bin/bash
set -euo pipefail
SCRIPT_DIR=$(realpath "$(dirname "$0")")
trap "popd >> /dev/null" EXIT
pushd "$SCRIPT_DIR/.." >> /dev/null || {
echo "Error: Failed to change directory to $SCRIPT_DIR/.."
exit 1
}
pnpm install
pnpm run build
rm -rf ./dist/openai-codex-*.tgz
pnpm pack --pack-destination ./dist
mv ./dist/openai-codex-*.tgz ./dist/codex.tgz
docker build -t codex -f "./Dockerfile" .

View file

@ -0,0 +1,453 @@
#!/usr/bin/env python3
"""Stage and optionally package the @openai/codex npm module."""
import argparse
import json
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
CODEX_CLI_ROOT = SCRIPT_DIR.parent
REPO_ROOT = CODEX_CLI_ROOT.parent
RESPONSES_API_PROXY_NPM_ROOT = REPO_ROOT / "codex-rs" / "responses-api-proxy" / "npm"
CODEX_SDK_ROOT = REPO_ROOT / "sdk" / "typescript"
CODEX_NPM_NAME = "@openai/codex"
# `npm_name` is the local optional-dependency alias consumed by `bin/codex.js`.
# The underlying package published to npm is always `@openai/codex`.
CODEX_PLATFORM_PACKAGES: dict[str, dict[str, str]] = {
"codex-linux-x64": {
"npm_name": "@openai/codex-linux-x64",
"npm_tag": "linux-x64",
"target_triple": "x86_64-unknown-linux-musl",
"os": "linux",
"cpu": "x64",
},
"codex-linux-arm64": {
"npm_name": "@openai/codex-linux-arm64",
"npm_tag": "linux-arm64",
"target_triple": "aarch64-unknown-linux-musl",
"os": "linux",
"cpu": "arm64",
},
"codex-darwin-x64": {
"npm_name": "@openai/codex-darwin-x64",
"npm_tag": "darwin-x64",
"target_triple": "x86_64-apple-darwin",
"os": "darwin",
"cpu": "x64",
},
"codex-darwin-arm64": {
"npm_name": "@openai/codex-darwin-arm64",
"npm_tag": "darwin-arm64",
"target_triple": "aarch64-apple-darwin",
"os": "darwin",
"cpu": "arm64",
},
"codex-win32-x64": {
"npm_name": "@openai/codex-win32-x64",
"npm_tag": "win32-x64",
"target_triple": "x86_64-pc-windows-msvc",
"os": "win32",
"cpu": "x64",
},
"codex-win32-arm64": {
"npm_name": "@openai/codex-win32-arm64",
"npm_tag": "win32-arm64",
"target_triple": "aarch64-pc-windows-msvc",
"os": "win32",
"cpu": "arm64",
},
}
PACKAGE_EXPANSIONS: dict[str, list[str]] = {
"codex": ["codex", *CODEX_PLATFORM_PACKAGES],
}
PACKAGE_NATIVE_COMPONENTS: dict[str, list[str]] = {
"codex": [],
"codex-linux-x64": ["codex", "rg"],
"codex-linux-arm64": ["codex", "rg"],
"codex-darwin-x64": ["codex", "rg"],
"codex-darwin-arm64": ["codex", "rg"],
"codex-win32-x64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"],
"codex-win32-arm64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"],
"codex-responses-api-proxy": ["codex-responses-api-proxy"],
"codex-sdk": [],
}
PACKAGE_TARGET_FILTERS: dict[str, str] = {
package_name: package_config["target_triple"]
for package_name, package_config in CODEX_PLATFORM_PACKAGES.items()
}
PACKAGE_CHOICES = tuple(PACKAGE_NATIVE_COMPONENTS)
COMPONENT_DEST_DIR: dict[str, str] = {
"codex": "codex",
"codex-responses-api-proxy": "codex-responses-api-proxy",
"codex-windows-sandbox-setup": "codex",
"codex-command-runner": "codex",
"rg": "path",
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build or stage the Codex CLI npm package.")
parser.add_argument(
"--package",
choices=PACKAGE_CHOICES,
default="codex",
help="Which npm package to stage (default: codex).",
)
parser.add_argument(
"--version",
help="Version number to write to package.json inside the staged package.",
)
parser.add_argument(
"--release-version",
help=(
"Version to stage for npm release."
),
)
parser.add_argument(
"--staging-dir",
type=Path,
help=(
"Directory to stage the package contents. Defaults to a new temporary directory "
"if omitted. The directory must be empty when provided."
),
)
parser.add_argument(
"--tmp",
dest="staging_dir",
type=Path,
help=argparse.SUPPRESS,
)
parser.add_argument(
"--pack-output",
type=Path,
help="Path where the generated npm tarball should be written.",
)
parser.add_argument(
"--vendor-src",
type=Path,
help="Directory containing pre-installed native binaries to bundle (vendor root).",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
package = args.package
version = args.version
release_version = args.release_version
if release_version:
if version and version != release_version:
raise RuntimeError("--version and --release-version must match when both are provided.")
version = release_version
if not version:
raise RuntimeError("Must specify --version or --release-version.")
staging_dir, created_temp = prepare_staging_dir(args.staging_dir)
try:
stage_sources(staging_dir, version, package)
vendor_src = args.vendor_src.resolve() if args.vendor_src else None
native_components = PACKAGE_NATIVE_COMPONENTS.get(package, [])
target_filter = PACKAGE_TARGET_FILTERS.get(package)
if native_components:
if vendor_src is None:
components_str = ", ".join(native_components)
raise RuntimeError(
"Native components "
f"({components_str}) required for package '{package}'. Provide --vendor-src "
"pointing to a directory containing pre-installed binaries."
)
copy_native_binaries(
vendor_src,
staging_dir,
native_components,
target_filter={target_filter} if target_filter else None,
)
if release_version:
staging_dir_str = str(staging_dir)
if package == "codex":
print(
f"Staged version {version} for release in {staging_dir_str}\n\n"
"Verify the CLI:\n"
f" node {staging_dir_str}/bin/codex.js --version\n"
f" node {staging_dir_str}/bin/codex.js --help\n\n"
)
elif package == "codex-responses-api-proxy":
print(
f"Staged version {version} for release in {staging_dir_str}\n\n"
"Verify the responses API proxy:\n"
f" node {staging_dir_str}/bin/codex-responses-api-proxy.js --help\n\n"
)
elif package in CODEX_PLATFORM_PACKAGES:
print(
f"Staged version {version} for release in {staging_dir_str}\n\n"
"Verify native payload contents:\n"
f" ls {staging_dir_str}/vendor\n\n"
)
else:
print(
f"Staged version {version} for release in {staging_dir_str}\n\n"
"Verify the SDK contents:\n"
f" ls {staging_dir_str}/dist\n"
" node -e \"import('./dist/index.js').then(() => console.log('ok'))\"\n\n"
)
else:
print(f"Staged package in {staging_dir}")
if args.pack_output is not None:
output_path = run_npm_pack(staging_dir, args.pack_output)
print(f"npm pack output written to {output_path}")
finally:
if created_temp:
# Preserve the staging directory for further inspection.
pass
return 0
def prepare_staging_dir(staging_dir: Path | None) -> tuple[Path, bool]:
if staging_dir is not None:
staging_dir = staging_dir.resolve()
staging_dir.mkdir(parents=True, exist_ok=True)
if any(staging_dir.iterdir()):
raise RuntimeError(f"Staging directory {staging_dir} is not empty.")
return staging_dir, False
temp_dir = Path(tempfile.mkdtemp(prefix="codex-npm-stage-"))
return temp_dir, True
def stage_sources(staging_dir: Path, version: str, package: str) -> None:
package_json: dict
package_json_path: Path | None = None
if package == "codex":
bin_dir = staging_dir / "bin"
bin_dir.mkdir(parents=True, exist_ok=True)
shutil.copy2(CODEX_CLI_ROOT / "bin" / "codex.js", bin_dir / "codex.js")
rg_manifest = CODEX_CLI_ROOT / "bin" / "rg"
if rg_manifest.exists():
shutil.copy2(rg_manifest, bin_dir / "rg")
readme_src = REPO_ROOT / "README.md"
if readme_src.exists():
shutil.copy2(readme_src, staging_dir / "README.md")
package_json_path = CODEX_CLI_ROOT / "package.json"
elif package in CODEX_PLATFORM_PACKAGES:
platform_package = CODEX_PLATFORM_PACKAGES[package]
platform_npm_tag = platform_package["npm_tag"]
platform_version = compute_platform_package_version(version, platform_npm_tag)
readme_src = REPO_ROOT / "README.md"
if readme_src.exists():
shutil.copy2(readme_src, staging_dir / "README.md")
with open(CODEX_CLI_ROOT / "package.json", "r", encoding="utf-8") as fh:
codex_package_json = json.load(fh)
package_json = {
"name": CODEX_NPM_NAME,
"version": platform_version,
"license": codex_package_json.get("license", "Apache-2.0"),
"os": [platform_package["os"]],
"cpu": [platform_package["cpu"]],
"files": ["vendor"],
"repository": codex_package_json.get("repository"),
}
engines = codex_package_json.get("engines")
if isinstance(engines, dict):
package_json["engines"] = engines
package_manager = codex_package_json.get("packageManager")
if isinstance(package_manager, str):
package_json["packageManager"] = package_manager
elif package == "codex-responses-api-proxy":
bin_dir = staging_dir / "bin"
bin_dir.mkdir(parents=True, exist_ok=True)
launcher_src = RESPONSES_API_PROXY_NPM_ROOT / "bin" / "codex-responses-api-proxy.js"
shutil.copy2(launcher_src, bin_dir / "codex-responses-api-proxy.js")
readme_src = RESPONSES_API_PROXY_NPM_ROOT / "README.md"
if readme_src.exists():
shutil.copy2(readme_src, staging_dir / "README.md")
package_json_path = RESPONSES_API_PROXY_NPM_ROOT / "package.json"
elif package == "codex-sdk":
package_json_path = CODEX_SDK_ROOT / "package.json"
stage_codex_sdk_sources(staging_dir)
else:
raise RuntimeError(f"Unknown package '{package}'.")
if package_json_path is not None:
with open(package_json_path, "r", encoding="utf-8") as fh:
package_json = json.load(fh)
package_json["version"] = version
if package == "codex":
package_json["files"] = ["bin"]
package_json["optionalDependencies"] = {
CODEX_PLATFORM_PACKAGES[platform_package]["npm_name"]: (
f"npm:{CODEX_NPM_NAME}@"
f"{compute_platform_package_version(version, CODEX_PLATFORM_PACKAGES[platform_package]['npm_tag'])}"
)
for platform_package in PACKAGE_EXPANSIONS["codex"]
if platform_package != "codex"
}
elif package == "codex-sdk":
scripts = package_json.get("scripts")
if isinstance(scripts, dict):
scripts.pop("prepare", None)
dependencies = package_json.get("dependencies")
if not isinstance(dependencies, dict):
dependencies = {}
dependencies[CODEX_NPM_NAME] = version
package_json["dependencies"] = dependencies
with open(staging_dir / "package.json", "w", encoding="utf-8") as out:
json.dump(package_json, out, indent=2)
out.write("\n")
def compute_platform_package_version(version: str, platform_tag: str) -> str:
# npm forbids republishing the same package name/version, so each
# platform-specific tarball needs a unique version string.
return f"{version}-{platform_tag}"
def run_command(cmd: list[str], cwd: Path | None = None) -> None:
print("+", " ".join(cmd))
subprocess.run(cmd, cwd=cwd, check=True)
def stage_codex_sdk_sources(staging_dir: Path) -> None:
package_root = CODEX_SDK_ROOT
run_command(["pnpm", "install", "--frozen-lockfile"], cwd=package_root)
run_command(["pnpm", "run", "build"], cwd=package_root)
dist_src = package_root / "dist"
if not dist_src.exists():
raise RuntimeError("codex-sdk build did not produce a dist directory.")
shutil.copytree(dist_src, staging_dir / "dist")
readme_src = package_root / "README.md"
if readme_src.exists():
shutil.copy2(readme_src, staging_dir / "README.md")
license_src = REPO_ROOT / "LICENSE"
if license_src.exists():
shutil.copy2(license_src, staging_dir / "LICENSE")
def copy_native_binaries(
vendor_src: Path,
staging_dir: Path,
components: list[str],
target_filter: set[str] | None = None,
) -> None:
vendor_src = vendor_src.resolve()
if not vendor_src.exists():
raise RuntimeError(f"Vendor source directory not found: {vendor_src}")
components_set = {component for component in components if component in COMPONENT_DEST_DIR}
if not components_set:
return
vendor_dest = staging_dir / "vendor"
if vendor_dest.exists():
shutil.rmtree(vendor_dest)
vendor_dest.mkdir(parents=True, exist_ok=True)
copied_targets: set[str] = set()
for target_dir in vendor_src.iterdir():
if not target_dir.is_dir():
continue
if target_filter is not None and target_dir.name not in target_filter:
continue
dest_target_dir = vendor_dest / target_dir.name
dest_target_dir.mkdir(parents=True, exist_ok=True)
copied_targets.add(target_dir.name)
for component in components_set:
dest_dir_name = COMPONENT_DEST_DIR.get(component)
if dest_dir_name is None:
continue
src_component_dir = target_dir / dest_dir_name
if not src_component_dir.exists():
raise RuntimeError(
f"Missing native component '{component}' in vendor source: {src_component_dir}"
)
dest_component_dir = dest_target_dir / dest_dir_name
if dest_component_dir.exists():
shutil.rmtree(dest_component_dir)
shutil.copytree(src_component_dir, dest_component_dir)
if target_filter is not None:
missing_targets = sorted(target_filter - copied_targets)
if missing_targets:
missing_list = ", ".join(missing_targets)
raise RuntimeError(f"Missing target directories in vendor source: {missing_list}")
def run_npm_pack(staging_dir: Path, output_path: Path) -> Path:
output_path = output_path.resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="codex-npm-pack-") as pack_dir_str:
pack_dir = Path(pack_dir_str)
stdout = subprocess.check_output(
["npm", "pack", "--json", "--pack-destination", str(pack_dir)],
cwd=staging_dir,
text=True,
)
try:
pack_output = json.loads(stdout)
except json.JSONDecodeError as exc:
raise RuntimeError("Failed to parse npm pack output.") from exc
if not pack_output:
raise RuntimeError("npm pack did not produce an output tarball.")
tarball_name = pack_output[0].get("filename") or pack_output[0].get("name")
if not tarball_name:
raise RuntimeError("Unable to determine npm pack output filename.")
tarball_path = pack_dir / tarball_name
if not tarball_path.exists():
raise RuntimeError(f"Expected npm pack output not found: {tarball_path}")
shutil.move(str(tarball_path), output_path)
return output_path
if __name__ == "__main__":
import sys
sys.exit(main())

View file

@ -0,0 +1,115 @@
#!/bin/bash
set -euo pipefail # Exit on error, undefined vars, and pipeline failures
IFS=$'\n\t' # Stricter word splitting
# Read allowed domains from file
ALLOWED_DOMAINS_FILE="/etc/codex/allowed_domains.txt"
if [ -f "$ALLOWED_DOMAINS_FILE" ]; then
ALLOWED_DOMAINS=()
while IFS= read -r domain; do
ALLOWED_DOMAINS+=("$domain")
done < "$ALLOWED_DOMAINS_FILE"
echo "Using domains from file: ${ALLOWED_DOMAINS[*]}"
else
# Fallback to default domains
ALLOWED_DOMAINS=("api.openai.com")
echo "Domains file not found, using default: ${ALLOWED_DOMAINS[*]}"
fi
# Ensure we have at least one domain
if [ ${#ALLOWED_DOMAINS[@]} -eq 0 ]; then
echo "ERROR: No allowed domains specified"
exit 1
fi
# Flush existing rules and delete existing ipsets
iptables -F
iptables -X
iptables -t nat -F
iptables -t nat -X
iptables -t mangle -F
iptables -t mangle -X
ipset destroy allowed-domains 2>/dev/null || true
# First allow DNS and localhost before any restrictions
# Allow outbound DNS
iptables -A OUTPUT -p udp --dport 53 -j ACCEPT
# Allow inbound DNS responses
iptables -A INPUT -p udp --sport 53 -j ACCEPT
# Allow localhost
iptables -A INPUT -i lo -j ACCEPT
iptables -A OUTPUT -o lo -j ACCEPT
# Create ipset with CIDR support
ipset create allowed-domains hash:net
# Resolve and add other allowed domains
for domain in "${ALLOWED_DOMAINS[@]}"; do
echo "Resolving $domain..."
ips=$(dig +short A "$domain")
if [ -z "$ips" ]; then
echo "ERROR: Failed to resolve $domain"
exit 1
fi
while read -r ip; do
if [[ ! "$ip" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
echo "ERROR: Invalid IP from DNS for $domain: $ip"
exit 1
fi
echo "Adding $ip for $domain"
ipset add allowed-domains "$ip"
done < <(echo "$ips")
done
# Get host IP from default route
HOST_IP=$(ip route | grep default | cut -d" " -f3)
if [ -z "$HOST_IP" ]; then
echo "ERROR: Failed to detect host IP"
exit 1
fi
HOST_NETWORK=$(echo "$HOST_IP" | sed "s/\.[0-9]*$/.0\/24/")
echo "Host network detected as: $HOST_NETWORK"
# Set up remaining iptables rules
iptables -A INPUT -s "$HOST_NETWORK" -j ACCEPT
iptables -A OUTPUT -d "$HOST_NETWORK" -j ACCEPT
# Set default policies to DROP first
iptables -P INPUT DROP
iptables -P FORWARD DROP
iptables -P OUTPUT DROP
# First allow established connections for already approved traffic
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
# Then allow only specific outbound traffic to allowed domains
iptables -A OUTPUT -m set --match-set allowed-domains dst -j ACCEPT
# Append final REJECT rules for immediate error responses
# For TCP traffic, send a TCP reset; for UDP, send ICMP port unreachable.
iptables -A INPUT -p tcp -j REJECT --reject-with tcp-reset
iptables -A INPUT -p udp -j REJECT --reject-with icmp-port-unreachable
iptables -A OUTPUT -p tcp -j REJECT --reject-with tcp-reset
iptables -A OUTPUT -p udp -j REJECT --reject-with icmp-port-unreachable
iptables -A FORWARD -p tcp -j REJECT --reject-with tcp-reset
iptables -A FORWARD -p udp -j REJECT --reject-with icmp-port-unreachable
echo "Firewall configuration complete"
echo "Verifying firewall rules..."
if curl --connect-timeout 5 https://example.com >/dev/null 2>&1; then
echo "ERROR: Firewall verification failed - was able to reach https://example.com"
exit 1
else
echo "Firewall verification passed - unable to reach https://example.com as expected"
fi
# Always verify OpenAI API access is working
if ! curl --connect-timeout 5 https://api.openai.com >/dev/null 2>&1; then
echo "ERROR: Firewall verification failed - unable to reach https://api.openai.com"
exit 1
else
echo "Firewall verification passed - able to reach https://api.openai.com as expected"
fi

View file

@ -0,0 +1,475 @@
#!/usr/bin/env python3
"""Install Codex native binaries (Rust CLI plus ripgrep helpers)."""
import argparse
from contextlib import contextmanager
import json
import os
import shutil
import subprocess
import tarfile
import tempfile
import zipfile
from dataclasses import dataclass
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
import sys
from typing import Iterable, Sequence
from urllib.parse import urlparse
from urllib.request import urlopen
SCRIPT_DIR = Path(__file__).resolve().parent
CODEX_CLI_ROOT = SCRIPT_DIR.parent
DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/17952349351" # rust-v0.40.0
VENDOR_DIR_NAME = "vendor"
RG_MANIFEST = CODEX_CLI_ROOT / "bin" / "rg"
BINARY_TARGETS = (
"x86_64-unknown-linux-musl",
"aarch64-unknown-linux-musl",
"x86_64-apple-darwin",
"aarch64-apple-darwin",
"x86_64-pc-windows-msvc",
"aarch64-pc-windows-msvc",
)
@dataclass(frozen=True)
class BinaryComponent:
artifact_prefix: str # matches the artifact filename prefix (e.g. codex-<target>.zst)
dest_dir: str # directory under vendor/<target>/ where the binary is installed
binary_basename: str # executable name inside dest_dir (before optional .exe)
targets: tuple[str, ...] | None = None # limit installation to specific targets
WINDOWS_TARGETS = tuple(target for target in BINARY_TARGETS if "windows" in target)
BINARY_COMPONENTS = {
"codex": BinaryComponent(
artifact_prefix="codex",
dest_dir="codex",
binary_basename="codex",
),
"codex-responses-api-proxy": BinaryComponent(
artifact_prefix="codex-responses-api-proxy",
dest_dir="codex-responses-api-proxy",
binary_basename="codex-responses-api-proxy",
),
"codex-windows-sandbox-setup": BinaryComponent(
artifact_prefix="codex-windows-sandbox-setup",
dest_dir="codex",
binary_basename="codex-windows-sandbox-setup",
targets=WINDOWS_TARGETS,
),
"codex-command-runner": BinaryComponent(
artifact_prefix="codex-command-runner",
dest_dir="codex",
binary_basename="codex-command-runner",
targets=WINDOWS_TARGETS,
),
}
RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [
("x86_64-unknown-linux-musl", "linux-x86_64"),
("aarch64-unknown-linux-musl", "linux-aarch64"),
("x86_64-apple-darwin", "macos-x86_64"),
("aarch64-apple-darwin", "macos-aarch64"),
("x86_64-pc-windows-msvc", "windows-x86_64"),
("aarch64-pc-windows-msvc", "windows-aarch64"),
]
RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS}
DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS]
# urllib.request.urlopen() defaults to no timeout (can hang indefinitely), which is painful in CI.
DOWNLOAD_TIMEOUT_SECS = 60
def _gha_enabled() -> bool:
# GitHub Actions supports "workflow commands" (e.g. ::group:: / ::error::) that make logs
# much easier to scan: groups collapse noisy sections and error annotations surface the
# failure in the UI without changing the actual exception/traceback output.
return os.environ.get("GITHUB_ACTIONS") == "true"
def _gha_escape(value: str) -> str:
# Workflow commands require percent/newline escaping.
return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
def _gha_error(*, title: str, message: str) -> None:
# Emit a GitHub Actions error annotation. This does not replace stdout/stderr logs; it just
# adds a prominent summary line to the job UI so the root cause is easier to spot.
if not _gha_enabled():
return
print(
f"::error title={_gha_escape(title)}::{_gha_escape(message)}",
flush=True,
)
@contextmanager
def _gha_group(title: str):
# Wrap a block in a collapsible log group on GitHub Actions. Outside of GHA this is a no-op
# so local output remains unchanged.
if _gha_enabled():
print(f"::group::{_gha_escape(title)}", flush=True)
try:
yield
finally:
if _gha_enabled():
print("::endgroup::", flush=True)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Install native Codex binaries.")
parser.add_argument(
"--workflow-url",
help=(
"GitHub Actions workflow URL that produced the artifacts. Defaults to a "
"known good run when omitted."
),
)
parser.add_argument(
"--component",
dest="components",
action="append",
choices=tuple(list(BINARY_COMPONENTS) + ["rg"]),
help=(
"Limit installation to the specified components."
" May be repeated. Defaults to codex, codex-windows-sandbox-setup,"
" codex-command-runner, and rg."
),
)
parser.add_argument(
"root",
nargs="?",
type=Path,
help=(
"Directory containing package.json for the staged package. If omitted, the "
"repository checkout is used."
),
)
return parser.parse_args()
def main() -> int:
args = parse_args()
codex_cli_root = (args.root or CODEX_CLI_ROOT).resolve()
vendor_dir = codex_cli_root / VENDOR_DIR_NAME
vendor_dir.mkdir(parents=True, exist_ok=True)
components = args.components or [
"codex",
"codex-windows-sandbox-setup",
"codex-command-runner",
"rg",
]
workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip()
if not workflow_url:
workflow_url = DEFAULT_WORKFLOW_URL
workflow_id = workflow_url.rstrip("/").split("/")[-1]
print(f"Downloading native artifacts from workflow {workflow_id}...")
with _gha_group(f"Download native artifacts from workflow {workflow_id}"):
with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str:
artifacts_dir = Path(artifacts_dir_str)
_download_artifacts(workflow_id, artifacts_dir)
install_binary_components(
artifacts_dir,
vendor_dir,
[BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS],
)
if "rg" in components:
with _gha_group("Fetch ripgrep binaries"):
print("Fetching ripgrep binaries...")
fetch_rg(vendor_dir, DEFAULT_RG_TARGETS, manifest_path=RG_MANIFEST)
print(f"Installed native dependencies into {vendor_dir}")
return 0
def fetch_rg(
vendor_dir: Path,
targets: Sequence[str] | None = None,
*,
manifest_path: Path,
) -> list[Path]:
"""Download ripgrep binaries described by the DotSlash manifest."""
if targets is None:
targets = DEFAULT_RG_TARGETS
if not manifest_path.exists():
raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}")
manifest = _load_manifest(manifest_path)
platforms = manifest.get("platforms", {})
vendor_dir.mkdir(parents=True, exist_ok=True)
targets = list(targets)
if not targets:
return []
task_configs: list[tuple[str, str, dict]] = []
for target in targets:
platform_key = RG_TARGET_TO_PLATFORM.get(target)
if platform_key is None:
raise ValueError(f"Unsupported ripgrep target '{target}'.")
platform_info = platforms.get(platform_key)
if platform_info is None:
raise RuntimeError(f"Platform '{platform_key}' not found in manifest {manifest_path}.")
task_configs.append((target, platform_key, platform_info))
results: dict[str, Path] = {}
max_workers = min(len(task_configs), max(1, (os.cpu_count() or 1)))
print("Installing ripgrep binaries for targets: " + ", ".join(targets))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_map = {
executor.submit(
_fetch_single_rg,
vendor_dir,
target,
platform_key,
platform_info,
manifest_path,
): target
for target, platform_key, platform_info in task_configs
}
for future in as_completed(future_map):
target = future_map[future]
try:
results[target] = future.result()
except Exception as exc:
_gha_error(
title="ripgrep install failed",
message=f"target={target} error={exc!r}",
)
raise RuntimeError(f"Failed to install ripgrep for target {target}.") from exc
print(f" installed ripgrep for {target}")
return [results[target] for target in targets]
def _download_artifacts(workflow_id: str, dest_dir: Path) -> None:
cmd = [
"gh",
"run",
"download",
"--dir",
str(dest_dir),
"--repo",
"openai/codex",
workflow_id,
]
subprocess.check_call(cmd)
def install_binary_components(
artifacts_dir: Path,
vendor_dir: Path,
selected_components: Sequence[BinaryComponent],
) -> None:
if not selected_components:
return
for component in selected_components:
component_targets = list(component.targets or BINARY_TARGETS)
print(
f"Installing {component.binary_basename} binaries for targets: "
+ ", ".join(component_targets)
)
max_workers = min(len(component_targets), max(1, (os.cpu_count() or 1)))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {
executor.submit(
_install_single_binary,
artifacts_dir,
vendor_dir,
target,
component,
): target
for target in component_targets
}
for future in as_completed(futures):
installed_path = future.result()
print(f" installed {installed_path}")
def _install_single_binary(
artifacts_dir: Path,
vendor_dir: Path,
target: str,
component: BinaryComponent,
) -> Path:
artifact_subdir = artifacts_dir / target
archive_name = _archive_name_for_target(component.artifact_prefix, target)
archive_path = artifact_subdir / archive_name
if not archive_path.exists():
raise FileNotFoundError(f"Expected artifact not found: {archive_path}")
dest_dir = vendor_dir / target / component.dest_dir
dest_dir.mkdir(parents=True, exist_ok=True)
binary_name = (
f"{component.binary_basename}.exe" if "windows" in target else component.binary_basename
)
dest = dest_dir / binary_name
dest.unlink(missing_ok=True)
extract_archive(archive_path, "zst", None, dest)
if "windows" not in target:
dest.chmod(0o755)
return dest
def _archive_name_for_target(artifact_prefix: str, target: str) -> str:
if "windows" in target:
return f"{artifact_prefix}-{target}.exe.zst"
return f"{artifact_prefix}-{target}.zst"
def _fetch_single_rg(
vendor_dir: Path,
target: str,
platform_key: str,
platform_info: dict,
manifest_path: Path,
) -> Path:
providers = platform_info.get("providers", [])
if not providers:
raise RuntimeError(f"No providers listed for platform '{platform_key}' in {manifest_path}.")
url = providers[0]["url"]
archive_format = platform_info.get("format", "zst")
archive_member = platform_info.get("path")
digest = platform_info.get("digest")
expected_size = platform_info.get("size")
dest_dir = vendor_dir / target / "path"
dest_dir.mkdir(parents=True, exist_ok=True)
is_windows = platform_key.startswith("win")
binary_name = "rg.exe" if is_windows else "rg"
dest = dest_dir / binary_name
with tempfile.TemporaryDirectory() as tmp_dir_str:
tmp_dir = Path(tmp_dir_str)
archive_filename = os.path.basename(urlparse(url).path)
download_path = tmp_dir / archive_filename
print(
f" downloading ripgrep for {target} ({platform_key}) from {url}",
flush=True,
)
try:
_download_file(url, download_path)
except Exception as exc:
_gha_error(
title="ripgrep download failed",
message=f"target={target} platform={platform_key} url={url} error={exc!r}",
)
raise RuntimeError(
"Failed to download ripgrep "
f"(target={target}, platform={platform_key}, format={archive_format}, "
f"expected_size={expected_size!r}, digest={digest!r}, url={url}, dest={download_path})."
) from exc
dest.unlink(missing_ok=True)
try:
extract_archive(download_path, archive_format, archive_member, dest)
except Exception as exc:
raise RuntimeError(
"Failed to extract ripgrep "
f"(target={target}, platform={platform_key}, format={archive_format}, "
f"member={archive_member!r}, url={url}, archive={download_path})."
) from exc
if not is_windows:
dest.chmod(0o755)
return dest
def _download_file(url: str, dest: Path) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
dest.unlink(missing_ok=True)
with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response, open(dest, "wb") as out:
shutil.copyfileobj(response, out)
def extract_archive(
archive_path: Path,
archive_format: str,
archive_member: str | None,
dest: Path,
) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
if archive_format == "zst":
output_path = archive_path.parent / dest.name
subprocess.check_call(
["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)]
)
shutil.move(str(output_path), dest)
return
if archive_format == "tar.gz":
if not archive_member:
raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.")
with tarfile.open(archive_path, "r:gz") as tar:
try:
member = tar.getmember(archive_member)
except KeyError as exc:
raise RuntimeError(
f"Entry '{archive_member}' not found in archive {archive_path}."
) from exc
tar.extract(member, path=archive_path.parent, filter="data")
extracted = archive_path.parent / archive_member
shutil.move(str(extracted), dest)
return
if archive_format == "zip":
if not archive_member:
raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.")
with zipfile.ZipFile(archive_path) as archive:
try:
with archive.open(archive_member) as src, open(dest, "wb") as out:
shutil.copyfileobj(src, out)
except KeyError as exc:
raise RuntimeError(
f"Entry '{archive_member}' not found in archive {archive_path}."
) from exc
return
raise RuntimeError(f"Unsupported archive format '{archive_format}'.")
def _load_manifest(manifest_path: Path) -> dict:
cmd = ["dotslash", "--", "parse", str(manifest_path)]
stdout = subprocess.check_output(cmd, text=True)
try:
manifest = json.loads(stdout)
except json.JSONDecodeError as exc:
raise RuntimeError(f"Invalid DotSlash manifest output from {manifest_path}.") from exc
if not isinstance(manifest, dict):
raise RuntimeError(
f"Unexpected DotSlash manifest structure for {manifest_path}: {type(manifest)!r}"
)
return manifest
if __name__ == "__main__":
import sys
sys.exit(main())

View file

@ -0,0 +1,95 @@
#!/bin/bash
set -e
# Usage:
# ./run_in_container.sh [--work_dir directory] "COMMAND"
#
# Examples:
# ./run_in_container.sh --work_dir project/code "ls -la"
# ./run_in_container.sh "echo Hello, world!"
# Default the work directory to WORKSPACE_ROOT_DIR if not provided.
WORK_DIR="${WORKSPACE_ROOT_DIR:-$(pwd)}"
# Default allowed domains - can be overridden with OPENAI_ALLOWED_DOMAINS env var
OPENAI_ALLOWED_DOMAINS="${OPENAI_ALLOWED_DOMAINS:-api.openai.com}"
# Parse optional flag.
if [ "$1" = "--work_dir" ]; then
if [ -z "$2" ]; then
echo "Error: --work_dir flag provided but no directory specified."
exit 1
fi
WORK_DIR="$2"
shift 2
fi
WORK_DIR=$(realpath "$WORK_DIR")
# Generate a unique container name based on the normalized work directory
CONTAINER_NAME="codex_$(echo "$WORK_DIR" | sed 's/\//_/g' | sed 's/[^a-zA-Z0-9_-]//g')"
# Define cleanup to remove the container on script exit, ensuring no leftover containers
cleanup() {
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
}
# Trap EXIT to invoke cleanup regardless of how the script terminates
trap cleanup EXIT
# Ensure a command is provided.
if [ "$#" -eq 0 ]; then
echo "Usage: $0 [--work_dir directory] \"COMMAND\""
exit 1
fi
# Check if WORK_DIR is set.
if [ -z "$WORK_DIR" ]; then
echo "Error: No work directory provided and WORKSPACE_ROOT_DIR is not set."
exit 1
fi
# Verify that OPENAI_ALLOWED_DOMAINS is not empty
if [ -z "$OPENAI_ALLOWED_DOMAINS" ]; then
echo "Error: OPENAI_ALLOWED_DOMAINS is empty."
exit 1
fi
# Kill any existing container for the working directory using cleanup(), centralizing removal logic.
cleanup
# Run the container with the specified directory mounted at the same path inside the container.
docker run --name "$CONTAINER_NAME" -d \
-e OPENAI_API_KEY \
--cap-add=NET_ADMIN \
--cap-add=NET_RAW \
-v "$WORK_DIR:/app$WORK_DIR" \
codex \
sleep infinity
# Write the allowed domains to a file in the container
docker exec --user root "$CONTAINER_NAME" bash -c "mkdir -p /etc/codex"
for domain in $OPENAI_ALLOWED_DOMAINS; do
# Validate domain format to prevent injection
if [[ ! "$domain" =~ ^[a-zA-Z0-9][a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]]; then
echo "Error: Invalid domain format: $domain"
exit 1
fi
echo "$domain" | docker exec --user root -i "$CONTAINER_NAME" bash -c "cat >> /etc/codex/allowed_domains.txt"
done
# Set proper permissions on the domains file
docker exec --user root "$CONTAINER_NAME" bash -c "chmod 444 /etc/codex/allowed_domains.txt && chown root:root /etc/codex/allowed_domains.txt"
# Initialize the firewall inside the container as root user
docker exec --user root "$CONTAINER_NAME" bash -c "/usr/local/bin/init_firewall.sh"
# Remove the firewall script after running it
docker exec --user root "$CONTAINER_NAME" bash -c "rm -f /usr/local/bin/init_firewall.sh"
# Execute the provided command in the container, ensuring it runs in the work directory.
# We use a parameterized bash command to safely handle the command and directory.
quoted_args=""
for arg in "$@"; do
quoted_args+=" $(printf '%q' "$arg")"
done
docker exec -it "$CONTAINER_NAME" bash -c "cd \"/app$WORK_DIR\" && codex --full-auto ${quoted_args}"