refactor: lay the repository out by role

local/ held six unrelated things under a name that stopped being true once it
became the production runtime: the service, the frontend, the tests, the ops
commands, the container definitions and the dependency lock, 65 files with
nothing to tell them apart.

  app/      the service: api/ routers, core/ for identity, database, models,
            prices and secret loading, and the worker, bootstrap and schema
  tests/    the twelve suites, no longer inside the shipped package
  ops/      backup, readiness, dependency audit, security summary
  infra/    Dockerfiles, gateway templates, ClamAV and storage configuration,
            the requirements and their hash lock
  web/      the Site, Kanban and portal pages with their scripts

deploy/Dockerfile.api now copies app/ alone, so the tests stop shipping to
production; the local image still carries them, because the suites run inside
the stack's network.

Five kinds of reference had to follow, and each was found by something different
rather than by reading. Imports of the form "from . import db" survived a rewrite
that only matched "from .db import". Tests kept relative imports of modules that
had left the package. A mock.patch target names its module in a string, where no
import rewriting can see it. The browser test resolves a fixture by path. And the
release gate's markers pointed at local/runtime.py and local/worker.py, which is
the decay its new marker test exists to catch — it caught it.

Verified from docker compose down -v: the stack starts, all six integration
suites, both browser suites and the twenty-nine unit tests pass.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

ops/__init__.py

@@ -0,0 +1 @@
+"""Operational commands: backup, readiness, audit, security summary."""

ops/backup.py

@@ -0,0 +1,128 @@
+"""Local PostgreSQL and clean-object backup with isolated restore verification."""
+import argparse
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+from uuid import uuid4
+
+ROOT = Path(__file__).resolve().parent.parent
+BACKUPS = ROOT / 'backups'
+
+def docker(script, *args, **kwargs):
+    return subprocess.run(['docker','compose','exec','-T','db','sh','-c',script,'sh',*args],
+                          cwd=ROOT,check=True,**kwargs)
+
+def checksum(path):
+    digest=hashlib.sha256()
+    with path.open('rb') as stream:
+        for block in iter(lambda:stream.read(1048576),b''):digest.update(block)
+    return digest.hexdigest()
+
+def compose_exec(service, *command, **kwargs):
+    return subprocess.run(['docker','compose','exec','-T',service,*command],
+                          cwd=ROOT,check=True,**kwargs)
+
+def create():
+    BACKUPS.mkdir(mode=0o700,exist_ok=True)
+    prefix='dtf-'+datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')+'-'+uuid4().hex[:8]
+    database=BACKUPS/(prefix+'.dump')
+    objects=BACKUPS/(prefix+'.objects.tar.gz')
+    manifest_path=BACKUPS/(prefix+'.manifest.json')
+    sidecar=BACKUPS/(prefix+'.manifest.sha256')
+    created=[]
+    try:
+        with database.open('xb') as stream:
+            created.append(database);database.chmod(0o600)
+            docker('pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom',stdout=stream)
+        with objects.open('xb') as stream:
+            created.append(objects);objects.chmod(0o600)
+            result=compose_exec('api','python','-m','local.storage_backup','export',
+                                stdout=stream,stderr=subprocess.PIPE)
+        summaries=[line.removeprefix('DTF_BACKUP_SUMMARY ') for line in
+                   result.stderr.decode().splitlines() if line.startswith('DTF_BACKUP_SUMMARY ')]
+        if len(summaries)!=1:
+            raise RuntimeError('Object backup did not return a valid summary')
+        manifest={'format':'dtf-local-backup-v2',
+                  'created_at':datetime.now(timezone.utc).isoformat(),
+                  'database':{'file':database.name,'sha256':checksum(database)},
+                  'objects':{'file':objects.name,'sha256':checksum(objects),
+                             **json.loads(summaries[0])}}
+        manifest_path.write_text(json.dumps(manifest,indent=2,sort_keys=True)+'\n')
+        created.append(manifest_path);manifest_path.chmod(0o600)
+        sidecar.write_text(checksum(manifest_path)+'\n')
+        created.append(sidecar);sidecar.chmod(0o600)
+    except Exception:
+        for path in reversed(created):path.unlink(missing_ok=True)
+        raise
+    print(manifest_path.relative_to(ROOT))
+    return manifest_path
+
+def verify_database(path):
+    if checksum(path)!=path.with_suffix('.sha256').read_text().strip():
+        raise ValueError('Backup checksum mismatch')
+    database='dtf_verify_'+uuid4().hex
+    docker('createdb -U "$POSTGRES_USER" "$1"',database)
+    try:
+        with path.open('rb') as stream:
+            docker('pg_restore -U "$POSTGRES_USER" -d "$1" --exit-on-error --no-owner --no-privileges',database,stdin=stream)
+        result=docker('psql -U "$POSTGRES_USER" -d "$1" -At -v ON_ERROR_STOP=1 -c "SELECT json_build_object(\'orders\',(SELECT count(*) FROM dtf_local.orders),\'uploads\',(SELECT count(*) FROM dtf_local.uploads),\'accounts\',(SELECT count(*) FROM dtf_local.accounts),\'history\',(SELECT count(*) FROM dtf_local.movements));"',database,capture_output=True,text=True)
+        print('PASS: backup restored to isolated database; restored counts '+result.stdout.strip())
+    finally:
+        # Only the freshly created UUID-named verification database is removed.
+        docker('dropdb -U "$POSTGRES_USER" "$1"',database)
+    print('Removed temporary verification database. Active database was untouched.')
+
+def bundle_file(manifest_path, name):
+    if not isinstance(name,str) or Path(name).name!=name:
+        raise ValueError('Backup manifest contains an invalid filename')
+    path=(manifest_path.parent/name).resolve()
+    if path.parent!=BACKUPS.resolve():
+        raise ValueError('Backup manifest references a file outside backups/')
+    return path
+
+def verify(path):
+    path=path.resolve()
+    if path.parent!=BACKUPS.resolve():
+        raise ValueError('Choose a backup generated in this repository backups/ directory')
+    if path.suffix=='.dump':
+        return verify_database(path)
+    if not path.name.endswith('.manifest.json'):
+        raise ValueError('Choose a v2 .manifest.json or legacy .dump backup')
+    sidecar=path.with_name(path.name.removesuffix('.json')+'.sha256')
+    if checksum(path)!=sidecar.read_text().strip():
+        raise ValueError('Backup manifest checksum mismatch')
+    manifest=json.loads(path.read_text())
+    if manifest.get('format')!='dtf-local-backup-v2':
+        raise ValueError('Unsupported backup format')
+    database=bundle_file(path,manifest.get('database',{}).get('file'))
+    objects=bundle_file(path,manifest.get('objects',{}).get('file'))
+    if checksum(database)!=manifest['database'].get('sha256') or checksum(objects)!=manifest['objects'].get('sha256'):
+        raise ValueError('Backup bundle checksum mismatch')
+    # Reuse the legacy database verifier with a temporary matching sidecar.
+    legacy_sidecar=database.with_suffix('.sha256')
+    had_legacy=legacy_sidecar.exists()
+    previous=legacy_sidecar.read_bytes() if had_legacy else None
+    legacy_sidecar.write_text(manifest['database']['sha256']+'\n');legacy_sidecar.chmod(0o600)
+    try:
+        verify_database(database)
+    finally:
+        if had_legacy:legacy_sidecar.write_bytes(previous)
+        else:legacy_sidecar.unlink(missing_ok=True)
+    with objects.open('rb') as stream:
+        compose_exec('api','python','-m','local.storage_backup','verify',stdin=stream)
+    print('PASS: combined database and clean-object backup verified. Active data was untouched.')
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('command',choices=['create','verify','create-and-verify'])
+    parser.add_argument('path',nargs='?',type=Path,
+                        help='v2 manifest printed by create, or a legacy .dump')
+    args=parser.parse_args()
+    if args.command=='verify':
+        if not args.path:parser.error('verify requires the path printed by create')
+        verify(args.path)
+    else:
+        path=create()
+        if args.command=='create-and-verify':verify(path)

ops/dependency_audit.py

@@ -0,0 +1,24 @@
+"""Audit exact installed API packages, not a host-dependent re-resolution.
+
+Install pip-audit in an isolated environment; pass its executable as argv[1].
+Only package names/versions are sent to the public vulnerability database.
+"""
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+if __name__=='__main__':
+    packages=json.loads(subprocess.check_output(['docker','compose','exec','-T','api','pip','list','--format=json'],text=True))
+    requirements='\n'.join(p['name']+'=='+p['version'] for p in packages)+'\n'
+    destination=Path('output/security');destination.mkdir(parents=True,exist_ok=True)
+    (destination/'runtime-requirements.txt').write_text(requirements)
+    result=subprocess.run([sys.argv[1] if len(sys.argv)>1 else 'pip-audit','--no-deps','--disable-pip',
+        '--progress-spinner','off','-r',str(destination/'runtime-requirements.txt'),
+        '-f','json','-o',str(destination/'python-audit.json')])
+    if (destination/'python-audit.json').exists():
+        data=json.loads((destination/'python-audit.json').read_text())
+        for package in data['dependencies']:
+            for vulnerability in package.get('vulns',[]):
+                print(package['name'],package['version'],vulnerability['id'],'fix:',','.join(vulnerability['fix_versions']))
+    sys.exit(result.returncode)

ops/security_status.py

@@ -0,0 +1,31 @@
+"""Local alert triage: prints redacted counts; exits 1 when attention is required."""
+import json
+from datetime import datetime, timezone
+from app.core.db import connect
+from app.scanning import ClamAV
+
+def scanner_status():
+    try:
+        version=ClamAV().version()
+        signature_text=version.split('/',2)[2]
+        signature_at=datetime.strptime(signature_text,'%a %b %d %H:%M:%S %Y').replace(tzinfo=timezone.utc)
+        age_hours=max(0,(datetime.now(timezone.utc)-signature_at).total_seconds()/3600)
+        return {'available':True,'version':version,'signature_age_hours':round(age_hours,1),
+                'signatures_stale':age_hours>24*7}
+    except Exception:
+        return {'available':False,'version':None,'signature_age_hours':None,'signatures_stale':True}
+
+def status():
+    with connect() as c:
+        rows=c.execute("""SELECT event,count(*) AS count FROM dtf_local.security_events
+            WHERE created_at>now()-interval '24 hours' GROUP BY event ORDER BY event""").fetchall()
+        blocked=c.execute("SELECT scan_state,count(*) AS count FROM dtf_local.uploads WHERE purged_at IS NULL AND scan_state IN ('error','rejected') GROUP BY scan_state").fetchall()
+    counts={r['event']:r['count'] for r in rows}
+    scanner=scanner_status()
+    attention=bool(blocked or not scanner['available'] or scanner['signatures_stale'] or
+                   counts.get('rate_limit',0) or counts.get('scanner_worker_error',0) or
+                   counts.get('operator_login_failed',0)+counts.get('customer_login_failed',0)>=10)
+    return {'attention_required':attention,'scanner':scanner,'last_24h':counts,'blocked_artwork':blocked}
+
+if __name__=='__main__':
+    result=status();print(json.dumps(result,indent=2));raise SystemExit(1 if result['attention_required'] else 0)

ops/staging_readiness.py

@@ -0,0 +1,100 @@
+"""Validate non-secret staging decisions without contacting external services."""
+from pathlib import Path
+import re
+import sys
+from urllib.parse import urlparse
+
+REQUIRED = (
+    'APP_ENV',
+    'STAGING_APPROVED_BY',
+    'STAGING_PUBLIC_ORIGIN',
+    'STAGING_S3_ENDPOINT',
+    'STAGING_S3_BUCKET',
+    'STAGING_DATABASE_MODE',
+    'STAGING_SECRET_SOURCE',
+    'STAGING_BACKUP_DESTINATION',
+    'STAGING_FREIGHT_PROVIDER',
+    'STAGING_PAYMENT_PROVIDER',
+    'STAGING_ERP_PROVIDER',
+    'STAGING_WHATSAPP_PROVIDER',
+    'STAGING_ALERT_OWNER',
+    'STAGING_ROLLBACK_OWNER',
+)
+FORBIDDEN_NAME = re.compile(
+    r'(PASSWORD|TOKEN|SECRET|ACCESS_KEY|PRIVATE_KEY|CREDENTIAL)', re.IGNORECASE)
+PLACEHOLDERS = {'', 'todo', 'tbd', 'replace-me', 'changeme', 'unconfirmed'}
+LOCAL_HOSTS = {'localhost', '127.0.0.1', '::1', 'storage', 'db'}
+
+
+def read_config(path: Path) -> dict[str, str]:
+    values = {}
+    for number, raw in enumerate(path.read_text().splitlines(), 1):
+        line = raw.strip()
+        if not line or line.startswith('#'):
+            continue
+        if '=' not in line:
+            raise ValueError(f'{path}:{number}: expected NAME=value')
+        name, value = line.split('=', 1)
+        name = name.strip()
+        if not re.fullmatch(r'[A-Z][A-Z0-9_]*', name):
+            raise ValueError(f'{path}:{number}: invalid setting name')
+        if name in values:
+            raise ValueError(f'{path}:{number}: duplicate setting {name}')
+        if name != 'STAGING_SECRET_SOURCE' and FORBIDDEN_NAME.search(name):
+            raise ValueError(f'{path}:{number}: secrets must not be stored in this file ({name})')
+        values[name] = value.strip()
+    return values
+
+
+def validate(values: dict[str, str]) -> list[str]:
+    errors = []
+    for name in REQUIRED:
+        if values.get(name, '').lower() in PLACEHOLDERS:
+            errors.append(f'{name} is not decided')
+    if values.get('APP_ENV') != 'staging':
+        errors.append('APP_ENV must be staging')
+    for name in ('STAGING_PUBLIC_ORIGIN', 'STAGING_S3_ENDPOINT'):
+        endpoint = urlparse(values.get(name, ''))
+        if endpoint.scheme != 'https' or not endpoint.hostname:
+            errors.append(f'{name} must be an absolute HTTPS URL')
+        elif endpoint.hostname.lower() in LOCAL_HOSTS:
+            errors.append(f'{name} must not point to localhost or a Compose service')
+        if endpoint.path not in ('', '/') or endpoint.params or endpoint.query or endpoint.fragment:
+            errors.append(f'{name} must not include a path, query, or fragment')
+    storage_host = urlparse(values.get('STAGING_S3_ENDPOINT', '')).hostname or ''
+    if storage_host and not storage_host.endswith('.r2.cloudflarestorage.com'):
+        errors.append('STAGING_S3_ENDPOINT must be a Cloudflare R2 S3 API endpoint')
+    bucket = values.get('STAGING_S3_BUCKET', '')
+    if bucket and not re.fullmatch(r'[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]', bucket):
+        errors.append('STAGING_S3_BUCKET is not a valid S3 bucket name')
+    for name in ('STAGING_FREIGHT_PROVIDER', 'STAGING_PAYMENT_PROVIDER',
+                 'STAGING_ERP_PROVIDER', 'STAGING_WHATSAPP_PROVIDER'):
+        if values.get(name, '').lower() in {'fake', 'local', 'mock'}:
+            errors.append(f'{name} cannot select a local fake provider')
+    if values.get('STAGING_DATABASE_MODE') not in {'managed', 'dedicated-container'}:
+        errors.append('STAGING_DATABASE_MODE must be managed or dedicated-container')
+    if values.get('STAGING_SECRET_SOURCE') in {'env-file', 'repository', '.env'}:
+        errors.append('STAGING_SECRET_SOURCE must be an external secret-injection mechanism')
+    return errors
+
+
+def main(path: Path) -> int:
+    try:
+        errors = validate(read_config(path))
+    except (OSError, ValueError) as exc:
+        print(f'BLOCKED: {exc}')
+        return 2
+    if errors:
+        print('BLOCKED: staging inputs are incomplete or unsafe:')
+        for error in errors:
+            print(f'- {error}')
+        return 2
+    print('PASS: non-secret staging inputs are complete and structurally safe.')
+    print('No provider was contacted. This check does not authorize deployment.')
+    return 0
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        raise SystemExit('usage: python -m local.staging_readiness PATH')
+    raise SystemExit(main(Path(sys.argv[1])))

ops/storage_backup.py

@@ -0,0 +1,192 @@
+"""Stream clean MinIO artwork to an archive and verify it via an isolated prefix."""
+import hashlib
+import io
+import json
+import os
+import sys
+import tarfile
+from datetime import datetime, timezone
+from uuid import uuid4
+
+from app.adapters import LocalS3Storage, require_local
+from app.core.db import connect
+
+CHUNK = 8 * 1024 * 1024
+MAX_OBJECT = min(128 * 1024 * 1024, int(os.environ.get('SCAN_MAX_BYTES', '134217728')))
+MAX_TOTAL = int(os.environ.get('STORAGE_QUOTA_BYTES', '53687091200'))
+MAX_OBJECTS = 100000
+
+
+class DigestReader:
+    def __init__(self, stream):
+        self.stream = stream
+        self.digest = hashlib.sha256()
+        self.size = 0
+
+    def read(self, size=-1):
+        block = self.stream.read(size)
+        if block:
+            self.digest.update(block)
+            self.size += len(block)
+        return block
+
+
+def add_bytes(archive, name, content):
+    info = tarfile.TarInfo(name)
+    info.size = len(content)
+    info.mtime = int(datetime.now(timezone.utc).timestamp())
+    info.mode = 0o600
+    archive.addfile(info, io.BytesIO(content))
+
+
+def export_archive():
+    require_local()
+    storage = LocalS3Storage()
+    with connect() as c:
+        rows = c.execute("""SELECT id,object_key,size,created_at,expires_at
+            FROM dtf_local.uploads WHERE complete AND purged_at IS NULL
+            AND expires_at>now() AND scan_state='clean' ORDER BY object_key""").fetchall()
+    if len(rows) > MAX_OBJECTS:
+        raise RuntimeError('Object count exceeds the backup safety limit')
+    expected_total = sum(row['size'] for row in rows)
+    if expected_total > MAX_TOTAL:
+        raise RuntimeError('Object bytes exceed the backup safety limit')
+
+    objects = []
+    with tarfile.open(fileobj=sys.stdout.buffer, mode='w|gz', compresslevel=6) as archive:
+        for index, row in enumerate(rows):
+            if row['size'] <= 0 or row['size'] > MAX_OBJECT:
+                raise RuntimeError('A clean object has an invalid backup size')
+            response = storage.client.get_object(Bucket=storage.bucket, Key=row['object_key'])
+            if response['ContentLength'] != row['size']:
+                response['Body'].close()
+                raise RuntimeError('Stored object size differs from database metadata')
+            reader = DigestReader(response['Body'])
+            name = f'objects/{index:08d}'
+            info = tarfile.TarInfo(name)
+            info.size = row['size']
+            info.mtime = int(row['created_at'].timestamp())
+            info.mode = 0o600
+            try:
+                archive.addfile(info, reader)
+            finally:
+                response['Body'].close()
+            if reader.size != row['size']:
+                raise RuntimeError('Object changed while the backup was streaming')
+            objects.append({'archive_name': name, 'upload_id': str(row['id']),
+                            'object_key': row['object_key'], 'size': row['size'],
+                            'sha256': reader.digest.hexdigest(),
+                            'expires_at': row['expires_at'].isoformat()})
+        manifest = {'format': 'dtf-object-backup-v1',
+                    'created_at': datetime.now(timezone.utc).isoformat(),
+                    'source_bucket': storage.bucket, 'objects': objects,
+                    'object_count': len(objects), 'total_bytes': expected_total}
+        add_bytes(archive, 'manifest.json', json.dumps(manifest, sort_keys=True).encode())
+    print('DTF_BACKUP_SUMMARY '+json.dumps({'object_count': len(objects),
+          'total_bytes': expected_total}), file=sys.stderr)
+
+
+def upload_member(storage, source, size, key):
+    upload = storage.client.create_multipart_upload(
+        Bucket=storage.bucket, Key=key, ContentType='application/octet-stream')['UploadId']
+    parts = []
+    digest = hashlib.sha256()
+    remaining = size
+    try:
+        number = 1
+        while remaining:
+            block = source.read(min(CHUNK, remaining))
+            if not block:
+                raise ValueError('Archive object ended before its declared size')
+            digest.update(block)
+            result = storage.client.upload_part(Bucket=storage.bucket, Key=key,
+                UploadId=upload, PartNumber=number, Body=block, ContentLength=len(block))
+            parts.append({'PartNumber': number, 'ETag': result['ETag']})
+            remaining -= len(block)
+            number += 1
+        storage.client.complete_multipart_upload(Bucket=storage.bucket, Key=key,
+            UploadId=upload, MultipartUpload={'Parts': parts})
+    except Exception:
+        try:
+            storage.client.abort_multipart_upload(
+                Bucket=storage.bucket, Key=key, UploadId=upload)
+        except Exception:
+            # Preserve the original upload/read error; the verification-prefix
+            # cleanup below still removes any completed temporary object.
+            pass
+        raise
+    return digest.hexdigest()
+
+
+def object_digest(storage, key):
+    response = storage.client.get_object(Bucket=storage.bucket, Key=key)
+    digest = hashlib.sha256()
+    size = 0
+    try:
+        for block in response['Body'].iter_chunks(chunk_size=1024 * 1024):
+            digest.update(block)
+            size += len(block)
+    finally:
+        response['Body'].close()
+    return size, digest.hexdigest()
+
+
+def verify_archive():
+    require_local()
+    storage = LocalS3Storage()
+    verification = uuid4().hex
+    restored = []
+    observed = []
+    manifest = None
+    total = 0
+    try:
+        with tarfile.open(fileobj=sys.stdin.buffer, mode='r|gz') as archive:
+            for member in archive:
+                if not member.isfile():
+                    raise ValueError('Backup archive contains a non-file member')
+                source = archive.extractfile(member)
+                if source is None:
+                    raise ValueError('Backup archive member cannot be read')
+                if member.name == 'manifest.json':
+                    if manifest is not None or member.size > 1024 * 1024:
+                        raise ValueError('Invalid object-backup manifest')
+                    manifest = json.loads(source.read().decode())
+                    continue
+                expected_name = f'objects/{len(observed):08d}'
+                if member.name != expected_name or len(observed) >= MAX_OBJECTS:
+                    raise ValueError('Unexpected object-backup member')
+                if member.size <= 0 or member.size > MAX_OBJECT:
+                    raise ValueError('Object-backup member exceeds safety limits')
+                total += member.size
+                if total > MAX_TOTAL:
+                    raise ValueError('Object-backup total exceeds safety limits')
+                key = f'originals/restore-verification/{verification}/{len(observed):08d}'
+                digest = upload_member(storage, source, member.size, key)
+                restored.append(key)
+                observed.append({'archive_name': member.name, 'size': member.size,
+                                 'sha256': digest})
+        if not manifest or manifest.get('format') != 'dtf-object-backup-v1':
+            raise ValueError('Object-backup manifest is missing or unsupported')
+        expected = manifest.get('objects')
+        if manifest.get('object_count') != len(observed) or manifest.get('total_bytes') != total:
+            raise ValueError('Object-backup summary does not match archive contents')
+        if not isinstance(expected, list) or len(expected) != len(observed):
+            raise ValueError('Object-backup manifest count does not match')
+        for actual, recorded, key in zip(observed, expected, restored):
+            for field in ('archive_name', 'size', 'sha256'):
+                if actual[field] != recorded.get(field):
+                    raise ValueError('Object-backup checksum manifest does not match')
+            restored_size, restored_hash = object_digest(storage, key)
+            if restored_size != actual['size'] or restored_hash != actual['sha256']:
+                raise ValueError('Restored verification object differs from archive')
+        print(f'PASS: {len(observed)} clean objects ({total} bytes) restored and hashed in an isolated prefix.')
+    finally:
+        for key in restored:
+            storage.client.delete_object(Bucket=storage.bucket, Key=key)
+    print('Removed temporary verification objects. Active artwork was untouched.')
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2 or sys.argv[1] not in ('export', 'verify'):
+        raise SystemExit('usage: python -m local.storage_backup export|verify')
+    export_archive() if sys.argv[1] == 'export' else verify_archive()