fix(migrate): restore storage object xattrs lost in the server move
CI / verify (push) Has been cancelled
CI / verify (push) Has been cancelled
After the netralax.cloud -> netralax.de migration every storage object GET
returned HTTP 500 (ENODATA "The extended attribute does not exist"). Root
cause: the object bytes were copied but Supabase Storage (file backend,
v1.48.26) keeps each object's response metadata in Linux xattrs
(user.supabase.{content-type,cache-control,etag}); the copy did not preserve
them. The old server is gone, but the values survive in storage.objects.metadata,
so this script reconstructs the xattrs from the DB. Verified: public avatar GETs
went 500 -> 200 after running it; all 18 objects (avatars, banner, attachments)
restored, 0 files genuinely missing. Idempotent, touches no object bytes.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
# Incident fix (2026-06-02): after the netralax.cloud -> netralax.de move, every
|
||||
# storage object GET returned HTTP 500 with:
|
||||
# { "code": "ENODATA", "errno": 61, "message": "The extended attribute does not exist." }
|
||||
#
|
||||
# Root cause: the OBJECT BYTES were copied to the new server, but Supabase
|
||||
# Storage (supabase/storage-api:v1.48.26, file backend) keeps each object's
|
||||
# response metadata in Linux extended attributes (xattrs) on the version file:
|
||||
# user.supabase.content-type
|
||||
# user.supabase.cache-control
|
||||
# user.supabase.etag
|
||||
# The migration copy did not preserve xattrs, so storage's getObject() throws
|
||||
# ENODATA when it reads them. The OLD server is gone, so we cannot re-copy —
|
||||
# but every value we need is still in the database column storage.objects.metadata
|
||||
# (mimetype / cacheControl / eTag). This script reconstructs the missing xattrs
|
||||
# from that column. It is idempotent and only ADDS metadata xattrs; it never
|
||||
# touches object bytes.
|
||||
#
|
||||
# RUN ON THE NEW SERVER (needs /opt/supabase, docker, and root for setxattr):
|
||||
# sudo python3 04-restore-storage-xattrs.py # all buckets
|
||||
# sudo python3 04-restore-storage-xattrs.py --dry-run # show, change nothing
|
||||
# sudo python3 04-restore-storage-xattrs.py --bucket profile-avatars
|
||||
# After it finishes, no storage restart is needed (xattrs are read per request).
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
SUPABASE_DIR = "/opt/supabase"
|
||||
# Single-tenant self-hosted layout: <volume>/stub/stub/<bucket>/<name>/<version-file>
|
||||
STORAGE_ROOT = os.path.join(SUPABASE_DIR, "volumes/storage/stub/stub")
|
||||
|
||||
# DB metadata field -> (xattr name, default when the field is absent)
|
||||
XATTRS = [
|
||||
("mimetype", "user.supabase.content-type", "application/octet-stream"),
|
||||
("cacheControl", "user.supabase.cache-control", "no-cache"),
|
||||
("eTag", "user.supabase.etag", None), # None default => skip if missing
|
||||
]
|
||||
|
||||
|
||||
def fetch_objects():
|
||||
"""Return [(bucket_id, name, metadata_dict), ...] from storage.objects."""
|
||||
# Tab-separate so object names containing '|' can't break parsing.
|
||||
query = (
|
||||
"select bucket_id||chr(9)||name||chr(9)||coalesce(metadata::text,'{}') "
|
||||
"from storage.objects"
|
||||
)
|
||||
raw = subprocess.check_output(
|
||||
[
|
||||
"docker", "compose", "exec", "-T", "db",
|
||||
"psql", "-U", "postgres", "-d", "postgres", "-tAc", query,
|
||||
],
|
||||
cwd=SUPABASE_DIR,
|
||||
).decode()
|
||||
rows = []
|
||||
for line in raw.splitlines():
|
||||
line = line.rstrip("\r")
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = line.split("\t", 2)
|
||||
if len(parts) < 3:
|
||||
continue
|
||||
bucket, name, meta = parts
|
||||
try:
|
||||
md = json.loads(meta) if meta else {}
|
||||
except json.JSONDecodeError:
|
||||
md = {}
|
||||
rows.append((bucket, name, md))
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--bucket", help="only this bucket (e.g. profile-avatars)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="print, change nothing")
|
||||
args = ap.parse_args()
|
||||
|
||||
objects = fetch_objects()
|
||||
fixed = missing_dir = missing_file = 0
|
||||
|
||||
for bucket, name, md in objects:
|
||||
if args.bucket and bucket != args.bucket:
|
||||
continue
|
||||
objdir = os.path.join(STORAGE_ROOT, bucket, name)
|
||||
if not os.path.isdir(objdir):
|
||||
print("NO_DIR ", bucket, name)
|
||||
missing_dir += 1
|
||||
continue
|
||||
version_files = [
|
||||
os.path.join(objdir, f)
|
||||
for f in os.listdir(objdir)
|
||||
if os.path.isfile(os.path.join(objdir, f))
|
||||
]
|
||||
if not version_files:
|
||||
print("NO_FILE ", bucket, name)
|
||||
missing_file += 1
|
||||
continue
|
||||
for path in version_files:
|
||||
for field, xattr, default in XATTRS:
|
||||
value = md.get(field, default)
|
||||
if value is None:
|
||||
continue
|
||||
if args.dry_run:
|
||||
print(f" would set {xattr}={value!r} on {path}")
|
||||
else:
|
||||
os.setxattr(path, xattr, str(value).encode())
|
||||
fixed += 1
|
||||
print("OK ", bucket, name)
|
||||
|
||||
print(
|
||||
f"\n{'DRY-RUN: would fix' if args.dry_run else 'fixed'} {fixed} file(s); "
|
||||
f"{missing_dir} missing dir(s), {missing_file} empty object dir(s)."
|
||||
)
|
||||
if missing_dir or missing_file:
|
||||
print(
|
||||
"NOTE: objects with a missing dir/file have lost their bytes and "
|
||||
"cannot be recovered from xattrs — those are genuinely gone."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if os.geteuid() != 0 and "--dry-run" not in sys.argv:
|
||||
print("Re-run with sudo (setxattr needs root).", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
main()
|
||||
Reference in New Issue
Block a user