Linus GasserandClaude Opus 5.5
e7aaf18d2d
- use rustfs instead of minio and provide migration script - Schedule clean_pending_items/purge_deleted_items with celery beat - Give celery workers the S3 access key secret - Collabora admin access needs setting a password Following the [C4DT AI Policy](https://git.coopcloud.tech/c4dt/ai-policy/src/branch/main/AI-POLICY.md): #LLM_vibe #LLM_model Claude Opus 5.5 - [x] I take full responsibility for this code - [x] I understand every line of this code - [x] I tested this code thoroughly on my own computer / servers - [x] None of the commit messages were created using LLMs - [x] I will participate in any discussion on my own and will not use an LLM for the discussion Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
97 lines
3.1 KiB
Bash
Executable File
97 lines
3.1 KiB
Bash
Executable File
#!/bin/sh
|
|
set -e
|
|
|
|
# One-shot copy of all objects from the old MinIO service (deployed with
|
|
# compose.minio-migration.yml) into the RustFS object storage.
|
|
# Only the latest version of each object is copied, and only objects missing
|
|
# on RustFS: objects changed on RustFS since (e.g. edited documents) are never
|
|
# overwritten. Once a run completes, a marker object is written and further
|
|
# runs are refused, unless called with --force.
|
|
|
|
# Load secrets into environment
|
|
source /abra-entrypoint.sh -e
|
|
|
|
python - "$@" <<'PYEOF'
|
|
import os
|
|
import sys
|
|
|
|
import boto3
|
|
from botocore.exceptions import ClientError
|
|
|
|
bucket = os.environ.get("AWS_STORAGE_BUCKET_NAME", "drive-media-storage")
|
|
marker = "_minio-migration-done"
|
|
force = "--force" in sys.argv[1:]
|
|
|
|
|
|
def secret(name):
|
|
with open(f"/run/secrets/{name}") as f:
|
|
return f.read().strip()
|
|
|
|
|
|
def client(endpoint, access_key, secret_key):
|
|
return boto3.client(
|
|
"s3",
|
|
endpoint_url=endpoint,
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
region_name=os.environ.get("AWS_S3_REGION_NAME", "us-east-1"),
|
|
)
|
|
|
|
|
|
def exists(s3, key):
|
|
try:
|
|
s3.head_object(Bucket=bucket, Key=key)
|
|
return True
|
|
except ClientError as error:
|
|
if error.response["Error"]["Code"] in ("404", "NoSuchKey", "NotFound"):
|
|
return False
|
|
raise
|
|
|
|
|
|
# the old MinIO uses the minio_* secrets, RustFS the rustfs_* secrets
|
|
src = client(
|
|
os.environ.get("MINIO_MIGRATION_ENDPOINT_URL", "http://minio:9000"),
|
|
secret("minio_ru"),
|
|
secret("minio_rp"),
|
|
)
|
|
dst = client(
|
|
os.environ["AWS_S3_ENDPOINT_URL"],
|
|
os.environ["AWS_S3_ACCESS_KEY_ID"],
|
|
os.environ["AWS_S3_SECRET_ACCESS_KEY"],
|
|
)
|
|
|
|
if exists(dst, marker) and not force:
|
|
sys.exit(
|
|
"migrate-from-minio: the migration already completed, refusing to run again.\n"
|
|
"Objects deleted on RustFS since then would be copied back from MinIO.\n"
|
|
"Use 'migrate_from_minio --force' to run anyway (existing objects are never overwritten)."
|
|
)
|
|
|
|
try:
|
|
src.head_bucket(Bucket=bucket)
|
|
except Exception as error:
|
|
sys.exit(f"migrate-from-minio: cannot reach bucket '{bucket}' on minio: {error}")
|
|
|
|
copied = skipped = 0
|
|
for page in src.get_paginator("list_objects_v2").paginate(Bucket=bucket):
|
|
for obj in page.get("Contents", []):
|
|
key = obj["Key"]
|
|
# upload_fileobj is atomic: an object only exists once fully copied
|
|
if exists(dst, key):
|
|
skipped += 1
|
|
continue
|
|
source = src.get_object(Bucket=bucket, Key=key)
|
|
extra = {"ContentType": source.get("ContentType", "binary/octet-stream")}
|
|
if source.get("Metadata"):
|
|
extra["Metadata"] = source["Metadata"]
|
|
if source.get("ContentDisposition"):
|
|
extra["ContentDisposition"] = source["ContentDisposition"]
|
|
dst.upload_fileobj(source["Body"], bucket, key, ExtraArgs=extra)
|
|
copied += 1
|
|
if copied % 100 == 0:
|
|
print(f"migrate-from-minio: {copied} objects copied...")
|
|
|
|
dst.put_object(Bucket=bucket, Key=marker, Body=b"")
|
|
print(f"migrate-from-minio: done, {copied} copied, {skipped} already present")
|
|
PYEOF
|