Upgrade to drive v0.25.3, replace MinIO with RustFS
- use rustfs instead of minio and provide migration script - Schedule clean_pending_items/purge_deleted_items with celery beat - Give celery workers the S3 access key secret - Collabora admin access needs setting a password Following the [C4DT AI Policy](https://git.coopcloud.tech/c4dt/ai-policy/src/branch/main/AI-POLICY.md): #LLM_vibe #LLM_model Claude Opus 5.5 - [x] I take full responsibility for this code - [x] I understand every line of this code - [x] I tested this code thoroughly on my own computer / servers - [x] None of the commit messages were created using LLMs - [x] I will participate in any discussion on my own and will not use an LLM for the discussion Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
committed by
Linus Gasser
co-authored by
Claude Opus 5.5
parent
a09d912312
commit
e7aaf18d2d
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# One-shot copy of all objects from the old MinIO service (deployed with
|
||||
# compose.minio-migration.yml) into the RustFS object storage.
|
||||
# Only the latest version of each object is copied, and only objects missing
|
||||
# on RustFS: objects changed on RustFS since (e.g. edited documents) are never
|
||||
# overwritten. Once a run completes, a marker object is written and further
|
||||
# runs are refused, unless called with --force.
|
||||
|
||||
# Load secrets into environment
|
||||
source /abra-entrypoint.sh -e
|
||||
|
||||
python - "$@" <<'PYEOF'
|
||||
import os
|
||||
import sys
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
bucket = os.environ.get("AWS_STORAGE_BUCKET_NAME", "drive-media-storage")
|
||||
marker = "_minio-migration-done"
|
||||
force = "--force" in sys.argv[1:]
|
||||
|
||||
|
||||
def secret(name):
|
||||
with open(f"/run/secrets/{name}") as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def client(endpoint, access_key, secret_key):
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=os.environ.get("AWS_S3_REGION_NAME", "us-east-1"),
|
||||
)
|
||||
|
||||
|
||||
def exists(s3, key):
|
||||
try:
|
||||
s3.head_object(Bucket=bucket, Key=key)
|
||||
return True
|
||||
except ClientError as error:
|
||||
if error.response["Error"]["Code"] in ("404", "NoSuchKey", "NotFound"):
|
||||
return False
|
||||
raise
|
||||
|
||||
|
||||
# the old MinIO uses the minio_* secrets, RustFS the rustfs_* secrets
|
||||
src = client(
|
||||
os.environ.get("MINIO_MIGRATION_ENDPOINT_URL", "http://minio:9000"),
|
||||
secret("minio_ru"),
|
||||
secret("minio_rp"),
|
||||
)
|
||||
dst = client(
|
||||
os.environ["AWS_S3_ENDPOINT_URL"],
|
||||
os.environ["AWS_S3_ACCESS_KEY_ID"],
|
||||
os.environ["AWS_S3_SECRET_ACCESS_KEY"],
|
||||
)
|
||||
|
||||
if exists(dst, marker) and not force:
|
||||
sys.exit(
|
||||
"migrate-from-minio: the migration already completed, refusing to run again.\n"
|
||||
"Objects deleted on RustFS since then would be copied back from MinIO.\n"
|
||||
"Use 'migrate_from_minio --force' to run anyway (existing objects are never overwritten)."
|
||||
)
|
||||
|
||||
try:
|
||||
src.head_bucket(Bucket=bucket)
|
||||
except Exception as error:
|
||||
sys.exit(f"migrate-from-minio: cannot reach bucket '{bucket}' on minio: {error}")
|
||||
|
||||
copied = skipped = 0
|
||||
for page in src.get_paginator("list_objects_v2").paginate(Bucket=bucket):
|
||||
for obj in page.get("Contents", []):
|
||||
key = obj["Key"]
|
||||
# upload_fileobj is atomic: an object only exists once fully copied
|
||||
if exists(dst, key):
|
||||
skipped += 1
|
||||
continue
|
||||
source = src.get_object(Bucket=bucket, Key=key)
|
||||
extra = {"ContentType": source.get("ContentType", "binary/octet-stream")}
|
||||
if source.get("Metadata"):
|
||||
extra["Metadata"] = source["Metadata"]
|
||||
if source.get("ContentDisposition"):
|
||||
extra["ContentDisposition"] = source["ContentDisposition"]
|
||||
dst.upload_fileobj(source["Body"], bucket, key, ExtraArgs=extra)
|
||||
copied += 1
|
||||
if copied % 100 == 0:
|
||||
print(f"migrate-from-minio: {copied} objects copied...")
|
||||
|
||||
dst.put_object(Bucket=bucket, Key=marker, Body=b"")
|
||||
print(f"migrate-from-minio: done, {copied} copied, {skipped} already present")
|
||||
PYEOF
|
||||
Reference in New Issue
Block a user