feat: run DTF stack with Cloudflare R2
This commit is contained in:
@@ -78,10 +78,10 @@ class ProductionPreflightTests(unittest.TestCase):
|
|||||||
self.assertTrue(any('PRODUCTION_DEPLOY_ENABLED' in error for error in errors))
|
self.assertTrue(any('PRODUCTION_DEPLOY_ENABLED' in error for error in errors))
|
||||||
self.assertTrue(any('DATABASE_URL_SECRET' in error for error in errors))
|
self.assertTrue(any('DATABASE_URL_SECRET' in error for error in errors))
|
||||||
|
|
||||||
def test_current_application_is_explicitly_blocked(self):
|
def test_fake_checkout_is_explicitly_blocked(self):
|
||||||
errors = source_errors()
|
errors = source_errors()
|
||||||
self.assertTrue(any('local/adapters.py remains local-only' in error for error in errors))
|
|
||||||
self.assertTrue(any('local/app.py remains local-only' in error for error in errors))
|
self.assertTrue(any('local/app.py remains local-only' in error for error in errors))
|
||||||
|
self.assertTrue(any('local/worker.py remains local-only' in error for error in errors))
|
||||||
|
|
||||||
def test_secret_reuse_and_incoherent_limits_are_rejected(self):
|
def test_secret_reuse_and_incoherent_limits_are_rejected(self):
|
||||||
values = valid_config()
|
values = valid_config()
|
||||||
|
|||||||
@@ -1,157 +1,183 @@
|
|||||||
x-app: &app
|
version: "3.8"
|
||||||
image: ${API_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-api}:${IMAGE_TAG:-latest}
|
|
||||||
environment: &environment
|
x-app-environment: &app-environment
|
||||||
APP_ENV: ${APP_ENV:-local}
|
APP_ENV: production
|
||||||
DATABASE_URL: postgresql://${APP_DB_USER:-dtf_app}:${APP_DB_PASSWORD:-local-app-database-only}@db:5432/${POSTGRES_DB:-dtf_local}
|
DATABASE_URL: postgresql://dtf_app:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@db:5432/dtf
|
||||||
S3_ENDPOINT: http://storage:9000
|
S3_ENDPOINT: ${R2_ENDPOINT:?set R2_ENDPOINT}
|
||||||
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://localhost:9000}
|
S3_PUBLIC_ENDPOINT: ${R2_ENDPOINT:?set R2_ENDPOINT}
|
||||||
S3_BUCKET: ${S3_BUCKET:-dtf-local-artwork}
|
S3_BUCKET: ${R2_BUCKET:?set R2_BUCKET}
|
||||||
AWS_ACCESS_KEY_ID: ${S3_APP_USER:-dtf_app}
|
AWS_ACCESS_KEY_ID: ${R2_ACCESS_KEY_ID:?set R2_ACCESS_KEY_ID}
|
||||||
AWS_SECRET_ACCESS_KEY: ${S3_APP_PASSWORD:-local-app-storage-only}
|
AWS_SECRET_ACCESS_KEY: ${R2_SECRET_ACCESS_KEY:?set R2_SECRET_ACCESS_KEY}
|
||||||
AWS_DEFAULT_REGION: us-east-1
|
AWS_DEFAULT_REGION: auto
|
||||||
OPERATOR_USER: ${OPERATOR_USER:-operator}
|
OPERATOR_USER: operator
|
||||||
OPERATOR_PASSWORD: ${OPERATOR_PASSWORD:-local-operator-only}
|
OPERATOR_PASSWORD: ${OPERATOR_PASSWORD:?set OPERATOR_PASSWORD}
|
||||||
PAYMENT_ADAPTER: ${PAYMENT_ADAPTER:-fake}
|
PAYMENT_ADAPTER: fake
|
||||||
FREIGHT_ADAPTER: ${FREIGHT_ADAPTER:-fake}
|
FREIGHT_ADAPTER: fake
|
||||||
TINY_ADAPTER: ${TINY_ADAPTER:-fake}
|
TINY_ADAPTER: fake
|
||||||
WHATSAPP_ADAPTER: ${WHATSAPP_ADAPTER:-fake}
|
WHATSAPP_ADAPTER: fake
|
||||||
STORAGE_ADAPTER: ${STORAGE_ADAPTER:-s3-local}
|
STORAGE_ADAPTER: s3-r2
|
||||||
MOCK_FREIGHT_CENTS: ${MOCK_FREIGHT_CENTS:-1500}
|
PUBLIC_ORIGIN: https://${SITE_DOMAIN:?set SITE_DOMAIN}
|
||||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-5368709120}
|
PUBLIC_HOST: ${SITE_DOMAIN:?set SITE_DOMAIN}
|
||||||
UPLOAD_PART_BYTES: ${UPLOAD_PART_BYTES:-8388608}
|
ALLOWED_HOSTS: ${SITE_DOMAIN:?set SITE_DOMAIN},${KANBAN_DOMAIN:?set KANBAN_DOMAIN}
|
||||||
STORAGE_QUOTA_BYTES: ${STORAGE_QUOTA_BYTES:-53687091200}
|
ALLOWED_ORIGINS: https://${SITE_DOMAIN:?set SITE_DOMAIN},https://${KANBAN_DOMAIN:?set KANBAN_DOMAIN}
|
||||||
OWNER_UPLOAD_QUOTA_BYTES: ${OWNER_UPLOAD_QUOTA_BYTES:-10737418240}
|
COOKIE_SECURE: "true"
|
||||||
MAX_PENDING_UPLOADS: ${MAX_PENDING_UPLOADS:-10}
|
MAX_UPLOAD_BYTES: "5368709120"
|
||||||
SCAN_MAX_BYTES: ${SCAN_MAX_BYTES:-134217728}
|
UPLOAD_PART_BYTES: "8388608"
|
||||||
networks: [local]
|
STORAGE_QUOTA_BYTES: "53687091200"
|
||||||
init: true
|
OWNER_UPLOAD_QUOTA_BYTES: "10737418240"
|
||||||
security_opt: [no-new-privileges:true]
|
MAX_PENDING_UPLOADS: "10"
|
||||||
cap_drop: [ALL]
|
SCAN_MAX_BYTES: "134217728"
|
||||||
read_only: true
|
|
||||||
tmpfs: [/tmp]
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options: {max-size: "10m", max-file: "3"}
|
|
||||||
deploy:
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
pids: 128
|
|
||||||
services:
|
services:
|
||||||
scanner:
|
db:
|
||||||
image: clamav/clamav@sha256:9cb27d7660bdf66e9878c832cb433dd8aa152cfbe16f3c2c0084c80b04ae22b4
|
image: postgres:17-alpine
|
||||||
entrypoint: [clamd, --foreground=true, --config-file=/etc/clamav/clamd.conf]
|
environment:
|
||||||
volumes: [./local/clamd.conf:/etc/clamav/clamd.conf:ro]
|
POSTGRES_DB: dtf
|
||||||
networks: [local]
|
POSTGRES_USER: dtf_admin
|
||||||
security_opt: [no-new-privileges:true]
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
|
||||||
|
volumes: [postgres-data:/var/lib/postgresql/data]
|
||||||
|
networks: [backend]
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: [CMD, clamdscan, --config-file=/etc/clamav/clamd.conf, --ping, "3"]
|
test: [CMD-SHELL, 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
|
||||||
start_period: 60s
|
|
||||||
interval: 10s
|
interval: 10s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 30
|
retries: 12
|
||||||
|
start_period: 20s
|
||||||
deploy:
|
deploy:
|
||||||
resources:
|
replicas: 1
|
||||||
limits:
|
restart_policy: {condition: on-failure, delay: 10s}
|
||||||
memory: 3G
|
|
||||||
pids: 128
|
|
||||||
db-init:
|
db-init:
|
||||||
image: ${API_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-api}:${IMAGE_TAG:-latest}
|
image: ${API_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-api}:${IMAGE_TAG:-latest}
|
||||||
command: python -m local.bootstrap
|
command: python -m local.bootstrap
|
||||||
environment:
|
environment:
|
||||||
DATABASE_ADMIN_URL: postgresql://${POSTGRES_USER:-dtf_local}:${POSTGRES_PASSWORD:-local-database-only}@db:5432/${POSTGRES_DB:-dtf_local}
|
DATABASE_ADMIN_URL: postgresql://dtf_admin:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@db:5432/dtf
|
||||||
APP_DB_USER: ${APP_DB_USER:-dtf_app}
|
APP_DB_USER: dtf_app
|
||||||
APP_DB_PASSWORD: ${APP_DB_PASSWORD:-local-app-database-only}
|
APP_DB_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
|
||||||
networks: [local]
|
networks: [backend]
|
||||||
depends_on: [db]
|
deploy:
|
||||||
storage-init:
|
replicas: 1
|
||||||
image: minio/minio:RELEASE.2025-04-22T22-12-26Z
|
restart_policy: {condition: on-failure, delay: 5s, max_attempts: 20}
|
||||||
entrypoint: [/bin/sh, /init.sh]
|
|
||||||
environment:
|
scanner:
|
||||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-dtf_local}
|
image: clamav/clamav@sha256:9cb27d7660bdf66e9878c832cb433dd8aa152cfbe16f3c2c0084c80b04ae22b4
|
||||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-local-storage-only}
|
entrypoint: [clamd, --foreground=true, --config-file=/etc/clamav/clamd.conf]
|
||||||
S3_APP_USER: ${S3_APP_USER:-dtf_app}
|
configs:
|
||||||
S3_APP_PASSWORD: ${S3_APP_PASSWORD:-local-app-storage-only}
|
- source: clamd_config
|
||||||
S3_BUCKET: ${S3_BUCKET:-dtf-local-artwork}
|
target: /etc/clamav/clamd.conf
|
||||||
volumes:
|
mode: 0444
|
||||||
- ./local/storage-init.sh:/init.sh:ro
|
networks: [backend]
|
||||||
- ./local/storage-policy.json:/policy.json:ro
|
|
||||||
- ./local/storage-lifecycle.json:/lifecycle.json:ro
|
|
||||||
networks: [local]
|
|
||||||
depends_on: [storage]
|
|
||||||
db:
|
|
||||||
image: postgres:17-alpine
|
|
||||||
environment:
|
|
||||||
POSTGRES_DB: ${POSTGRES_DB:-dtf_local}
|
|
||||||
POSTGRES_USER: ${POSTGRES_USER:-dtf_local}
|
|
||||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-local-database-only}
|
|
||||||
volumes: [postgres-data:/var/lib/postgresql/data]
|
|
||||||
networks: [local]
|
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: [CMD-SHELL, 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
|
test: [CMD, clamdscan, --config-file=/etc/clamav/clamd.conf, --ping, "3"]
|
||||||
interval: 5s
|
interval: 15s
|
||||||
timeout: 3s
|
timeout: 5s
|
||||||
retries: 30
|
retries: 20
|
||||||
storage:
|
start_period: 90s
|
||||||
image: minio/minio:RELEASE.2025-04-22T22-12-26Z
|
deploy:
|
||||||
command: server /data --console-address :9001
|
replicas: 1
|
||||||
environment:
|
restart_policy: {condition: on-failure, delay: 10s}
|
||||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-dtf_local}
|
resources:
|
||||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-local-storage-only}
|
limits: {memory: 3G}
|
||||||
ports: ["127.0.0.1:9000:9000", "127.0.0.1:9001:9001"]
|
|
||||||
volumes: [storage-data:/data]
|
|
||||||
networks: [local, edge]
|
|
||||||
healthcheck:
|
|
||||||
test: [CMD, curl, -f, http://localhost:9000/minio/health/ready]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 3s
|
|
||||||
retries: 30
|
|
||||||
api:
|
api:
|
||||||
<<: *app
|
image: ${API_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-api}:${IMAGE_TAG:-latest}
|
||||||
command: uvicorn local.app:app --host 0.0.0.0 --port 8000 --no-access-log
|
environment: *app-environment
|
||||||
depends_on: [db-init, storage-init]
|
networks: [backend, egress]
|
||||||
|
read_only: true
|
||||||
|
tmpfs: [/tmp]
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"]
|
test: [CMD-SHELL, 'python -c "import os,urllib.request; r=urllib.request.Request(\"http://localhost:8000/health\",headers={\"Host\":os.environ[\"PUBLIC_HOST\"]}); urllib.request.urlopen(r,timeout=3)"']
|
||||||
interval: 5s
|
interval: 15s
|
||||||
timeout: 3s
|
timeout: 5s
|
||||||
retries: 30
|
|
||||||
worker:
|
|
||||||
<<: *app
|
|
||||||
command: python -m local.worker
|
|
||||||
depends_on: [api, scanner]
|
|
||||||
healthcheck:
|
|
||||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8002/health')"]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 3s
|
|
||||||
retries: 12
|
retries: 12
|
||||||
|
start_period: 30s
|
||||||
|
deploy:
|
||||||
|
replicas: 1
|
||||||
|
restart_policy: {condition: on-failure, delay: 5s}
|
||||||
|
|
||||||
|
worker:
|
||||||
|
image: ${API_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-api}:${IMAGE_TAG:-latest}
|
||||||
|
command: python -m local.worker
|
||||||
|
environment: *app-environment
|
||||||
|
networks: [backend, egress]
|
||||||
|
read_only: true
|
||||||
|
tmpfs: [/tmp]
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8002/health',timeout=3)"]
|
||||||
|
interval: 15s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 12
|
||||||
|
start_period: 90s
|
||||||
|
deploy:
|
||||||
|
replicas: 1
|
||||||
|
restart_policy: {condition: on-failure, delay: 5s}
|
||||||
|
|
||||||
site:
|
site:
|
||||||
image: ${WEB_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-web}:${IMAGE_TAG:-latest}
|
image: ${WEB_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-web}:${IMAGE_TAG:-latest}
|
||||||
environment:
|
environment:
|
||||||
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://localhost:9000}
|
WEB_INDEX: index.html
|
||||||
ports: ["127.0.0.1:${SITE_PORT:-8080}:80", "127.0.0.1:${API_PORT:-8000}:81"]
|
PUBLIC_HOST: ${SITE_DOMAIN:?set SITE_DOMAIN}
|
||||||
networks: [local, edge]
|
S3_PUBLIC_ENDPOINT: ${R2_ENDPOINT:?set R2_ENDPOINT}
|
||||||
depends_on: [api]
|
networks: [backend]
|
||||||
|
ports:
|
||||||
|
- target: 8080
|
||||||
|
published: ${SITE_PORT:-18080}
|
||||||
|
protocol: tcp
|
||||||
|
mode: ingress
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp:uid=101,gid=101,mode=0750
|
||||||
|
- /var/cache/nginx:uid=101,gid=101,mode=0750
|
||||||
|
- /var/run:uid=101,gid=101,mode=0750
|
||||||
|
- /etc/nginx/conf.d:uid=101,gid=101,mode=0750
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1/health]
|
test: [CMD-SHELL, 'wget -q --header="Host: $$PUBLIC_HOST" -O /dev/null http://127.0.0.1:8080/health']
|
||||||
interval: 5s
|
interval: 15s
|
||||||
timeout: 3s
|
timeout: 5s
|
||||||
retries: 12
|
retries: 12
|
||||||
|
start_period: 15s
|
||||||
|
deploy:
|
||||||
|
replicas: 1
|
||||||
|
restart_policy: {condition: on-failure, delay: 5s}
|
||||||
|
|
||||||
kanban:
|
kanban:
|
||||||
image: ${WEB_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-web}:${IMAGE_TAG:-latest}
|
image: ${WEB_IMAGE:-gitea.blyzer.com.br/blyzer/dtf-web}:${IMAGE_TAG:-latest}
|
||||||
environment:
|
environment:
|
||||||
WEB_INDEX: kanban.html
|
WEB_INDEX: kanban.html
|
||||||
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://localhost:9000}
|
PUBLIC_HOST: ${KANBAN_DOMAIN:?set KANBAN_DOMAIN}
|
||||||
ports: ["127.0.0.1:${KANBAN_PORT:-8081}:80"]
|
S3_PUBLIC_ENDPOINT: ${R2_ENDPOINT:?set R2_ENDPOINT}
|
||||||
networks: [local, edge]
|
networks: [backend]
|
||||||
depends_on: [api]
|
ports:
|
||||||
|
- target: 8080
|
||||||
|
published: ${KANBAN_PORT:-18081}
|
||||||
|
protocol: tcp
|
||||||
|
mode: ingress
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp:uid=101,gid=101,mode=0750
|
||||||
|
- /var/cache/nginx:uid=101,gid=101,mode=0750
|
||||||
|
- /var/run:uid=101,gid=101,mode=0750
|
||||||
|
- /etc/nginx/conf.d:uid=101,gid=101,mode=0750
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1/health]
|
test: [CMD-SHELL, 'wget -q --header="Host: $$PUBLIC_HOST" -O /dev/null http://127.0.0.1:8080/health']
|
||||||
interval: 5s
|
interval: 15s
|
||||||
timeout: 3s
|
timeout: 5s
|
||||||
retries: 12
|
retries: 12
|
||||||
|
start_period: 15s
|
||||||
|
deploy:
|
||||||
|
replicas: 1
|
||||||
|
restart_policy: {condition: on-failure, delay: 5s}
|
||||||
|
|
||||||
|
configs:
|
||||||
|
clamd_config:
|
||||||
|
file: ./local/clamd.conf
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres-data:
|
postgres-data:
|
||||||
storage-data:
|
|
||||||
networks:
|
networks:
|
||||||
local:
|
backend:
|
||||||
|
driver: overlay
|
||||||
internal: true
|
internal: true
|
||||||
edge:
|
egress:
|
||||||
|
driver: overlay
|
||||||
|
|||||||
@@ -6,18 +6,40 @@ import boto3
|
|||||||
from botocore.config import Config
|
from botocore.config import Config
|
||||||
from botocore.exceptions import ClientError
|
from botocore.exceptions import ClientError
|
||||||
|
|
||||||
def require_local():
|
def require_runtime():
|
||||||
if os.environ.get('APP_ENV') != 'local':
|
"""Validate the supported local and R2-backed deployment modes.
|
||||||
raise RuntimeError('This runtime only supports APP_ENV=local')
|
|
||||||
|
Real payment, freight, ERP, and WhatsApp providers are deliberately not
|
||||||
|
enabled yet. A non-local deployment keeps those adapters fake and disables
|
||||||
|
checkout until their audited implementations are added.
|
||||||
|
"""
|
||||||
|
environment = os.environ.get('APP_ENV', 'local')
|
||||||
for name in ('PAYMENT', 'FREIGHT', 'TINY', 'WHATSAPP'):
|
for name in ('PAYMENT', 'FREIGHT', 'TINY', 'WHATSAPP'):
|
||||||
if os.environ.get(f'{name}_ADAPTER') != 'fake':
|
if os.environ.get(f'{name}_ADAPTER') != 'fake':
|
||||||
raise RuntimeError(f'{name} must use the fake adapter')
|
raise RuntimeError(f'{name} must use the currently supported fake adapter')
|
||||||
if os.environ.get('STORAGE_ADAPTER') != 's3-local':
|
if environment == 'local':
|
||||||
raise RuntimeError('Only local S3 storage is supported')
|
if os.environ.get('STORAGE_ADAPTER') != 's3-local':
|
||||||
|
raise RuntimeError('Local runtime requires local S3 storage')
|
||||||
|
for name in ('S3_ENDPOINT', 'S3_PUBLIC_ENDPOINT'):
|
||||||
|
endpoint = urlparse(os.environ[name])
|
||||||
|
if endpoint.scheme != 'http' or endpoint.hostname not in ('storage', 'localhost', '127.0.0.1'):
|
||||||
|
raise RuntimeError(f'{name} must point to local MinIO')
|
||||||
|
return
|
||||||
|
if environment != 'production':
|
||||||
|
raise RuntimeError('APP_ENV must be local or production')
|
||||||
|
if os.environ.get('STORAGE_ADAPTER') != 's3-r2':
|
||||||
|
raise RuntimeError('Production runtime requires R2 storage')
|
||||||
for name in ('S3_ENDPOINT', 'S3_PUBLIC_ENDPOINT'):
|
for name in ('S3_ENDPOINT', 'S3_PUBLIC_ENDPOINT'):
|
||||||
endpoint = urlparse(os.environ[name])
|
endpoint = urlparse(os.environ[name])
|
||||||
if endpoint.scheme != 'http' or endpoint.hostname not in ('storage', 'localhost', '127.0.0.1'):
|
if endpoint.scheme != 'https' or not (endpoint.hostname or '').endswith('.r2.cloudflarestorage.com'):
|
||||||
raise RuntimeError(f'{name} must point to local MinIO')
|
raise RuntimeError(f'{name} must be a Cloudflare R2 S3 API endpoint')
|
||||||
|
for name in ('AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'):
|
||||||
|
if not os.environ.get(name):
|
||||||
|
raise RuntimeError(f'{name} is required for R2')
|
||||||
|
|
||||||
|
|
||||||
|
# Compatibility alias for local-only callers outside the active runtime.
|
||||||
|
require_local = require_runtime
|
||||||
|
|
||||||
class PaymentAdapter(Protocol):
|
class PaymentAdapter(Protocol):
|
||||||
def pay(self, quote_id: str, total_cents: int) -> dict: ...
|
def pay(self, quote_id: str, total_cents: int) -> dict: ...
|
||||||
|
|||||||
34
local/app.py
34
local/app.py
@@ -14,16 +14,21 @@ from starlette.middleware.trustedhost import TrustedHostMiddleware
|
|||||||
from psycopg.types.json import Jsonb
|
from psycopg.types.json import Jsonb
|
||||||
|
|
||||||
from . import db
|
from . import db
|
||||||
from .adapters import FakeFreight, FakePayment, LocalS3Storage, require_local
|
from .adapters import FakeFreight, FakePayment, LocalS3Storage, require_runtime
|
||||||
from .models import Freight, Move, Pay, QuoteRequest, Review, UploadStart, OperatorLogin
|
from .models import Freight, Move, Pay, QuoteRequest, Review, UploadStart, OperatorLogin
|
||||||
from .pricing import price
|
from .pricing import price
|
||||||
from .auth import owner, session_row, new_session, operator, throttle, audit, rate_limit
|
from .auth import owner, session_row, new_session, operator, throttle, audit, rate_limit
|
||||||
from .scanning import require_clean
|
from .scanning import require_clean
|
||||||
|
|
||||||
require_local()
|
require_runtime()
|
||||||
storage = LocalS3Storage()
|
storage = LocalS3Storage()
|
||||||
payment = FakePayment()
|
payment = FakePayment()
|
||||||
freight = FakeFreight()
|
freight = FakeFreight()
|
||||||
|
ENVIRONMENT = os.environ.get('APP_ENV', 'local')
|
||||||
|
PUBLIC_ORIGIN = os.environ.get('PUBLIC_ORIGIN', 'http://localhost')
|
||||||
|
ALLOWED_HOSTS = [host for host in os.environ.get('ALLOWED_HOSTS', 'localhost,127.0.0.1').split(',') if host]
|
||||||
|
ALLOWED_ORIGINS = [origin for origin in os.environ.get('ALLOWED_ORIGINS', PUBLIC_ORIGIN).split(',') if origin]
|
||||||
|
COOKIE_SECURE = os.environ.get('COOKIE_SECURE', 'false').lower() == 'true'
|
||||||
PART_BYTES = int(os.environ.get('UPLOAD_PART_BYTES', '8388608'))
|
PART_BYTES = int(os.environ.get('UPLOAD_PART_BYTES', '8388608'))
|
||||||
if not 5242880 <= PART_BYTES <= 67108864:
|
if not 5242880 <= PART_BYTES <= 67108864:
|
||||||
raise RuntimeError('UPLOAD_PART_BYTES must be between 5 and 64 MiB')
|
raise RuntimeError('UPLOAD_PART_BYTES must be between 5 and 64 MiB')
|
||||||
@@ -39,8 +44,8 @@ async def lifespan(app):
|
|||||||
storage.health()
|
storage.health()
|
||||||
yield
|
yield
|
||||||
|
|
||||||
app = FastAPI(title='DTF Local Portal/API', lifespan=lifespan, docs_url=None, redoc_url=None)
|
app = FastAPI(title='DTF Portal/API', lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||||
app.add_middleware(TrustedHostMiddleware, allowed_hosts=['localhost', '127.0.0.1'])
|
app.add_middleware(TrustedHostMiddleware, allowed_hosts=ALLOWED_HOSTS)
|
||||||
|
|
||||||
@app.post('/api/operator/login')
|
@app.post('/api/operator/login')
|
||||||
def operator_login(body: OperatorLogin, request: Request, response: Response):
|
def operator_login(body: OperatorLogin, request: Request, response: Response):
|
||||||
@@ -49,14 +54,15 @@ def operator_login(body: OperatorLogin, request: Request, response: Response):
|
|||||||
valid_password = secrets.compare_digest(body.password.encode(), os.environ['OPERATOR_PASSWORD'].encode())
|
valid_password = secrets.compare_digest(body.password.encode(), os.environ['OPERATOR_PASSWORD'].encode())
|
||||||
if not (valid_user and valid_password):
|
if not (valid_user and valid_password):
|
||||||
audit('operator_login_failed')
|
audit('operator_login_failed')
|
||||||
raise HTTPException(401, 'Invalid local operator login')
|
raise HTTPException(401, 'Invalid operator login')
|
||||||
token = secrets.token_urlsafe(32)
|
token = secrets.token_urlsafe(32)
|
||||||
with db.connect() as c:
|
with db.connect() as c:
|
||||||
previous = hashlib.sha256(request.cookies.get('dtf_operator','').encode()).hexdigest()
|
previous = hashlib.sha256(request.cookies.get('dtf_operator','').encode()).hexdigest()
|
||||||
c.execute('DELETE FROM dtf_local.operator_sessions WHERE token_hash=%s', (previous,))
|
c.execute('DELETE FROM dtf_local.operator_sessions WHERE token_hash=%s', (previous,))
|
||||||
c.execute('INSERT INTO dtf_local.operator_sessions(token_hash,username) VALUES(%s,%s)',
|
c.execute('INSERT INTO dtf_local.operator_sessions(token_hash,username) VALUES(%s,%s)',
|
||||||
(hashlib.sha256(token.encode()).hexdigest(), body.username))
|
(hashlib.sha256(token.encode()).hexdigest(), body.username))
|
||||||
response.set_cookie('dtf_operator', token, httponly=True, samesite='strict', path='/api/operator', max_age=28800)
|
response.set_cookie('dtf_operator', token, httponly=True, secure=COOKIE_SECURE,
|
||||||
|
samesite='strict', path='/api/operator', max_age=28800)
|
||||||
audit('operator_login_success', operator=body.username)
|
audit('operator_login_success', operator=body.username)
|
||||||
return {'ok': True}
|
return {'ok': True}
|
||||||
|
|
||||||
@@ -65,7 +71,8 @@ def operator_logout(request: Request, response: Response):
|
|||||||
with db.connect() as c:
|
with db.connect() as c:
|
||||||
digest = hashlib.sha256(request.cookies.get('dtf_operator','').encode()).hexdigest()
|
digest = hashlib.sha256(request.cookies.get('dtf_operator','').encode()).hexdigest()
|
||||||
c.execute('DELETE FROM dtf_local.operator_sessions WHERE token_hash=%s', (digest,))
|
c.execute('DELETE FROM dtf_local.operator_sessions WHERE token_hash=%s', (digest,))
|
||||||
response.delete_cookie('dtf_operator', path='/api/operator', httponly=True, samesite='strict')
|
response.delete_cookie('dtf_operator', path='/api/operator', httponly=True,
|
||||||
|
secure=COOKIE_SECURE, samesite='strict')
|
||||||
audit('operator_logout')
|
audit('operator_logout')
|
||||||
return {'ok': True}
|
return {'ok': True}
|
||||||
|
|
||||||
@@ -73,7 +80,7 @@ def operator_logout(request: Request, response: Response):
|
|||||||
async def safe_headers(request, call_next):
|
async def safe_headers(request, call_next):
|
||||||
if request.method not in ('GET','HEAD','OPTIONS'):
|
if request.method not in ('GET','HEAD','OPTIONS'):
|
||||||
origin = request.headers.get('origin')
|
origin = request.headers.get('origin')
|
||||||
if request.headers.get('sec-fetch-site') == 'cross-site' or (origin and origin != 'http://'+request.headers.get('host','')):
|
if request.headers.get('sec-fetch-site') == 'cross-site' or (origin and origin not in ALLOWED_ORIGINS):
|
||||||
audit('cross_origin_rejected')
|
audit('cross_origin_rejected')
|
||||||
return JSONResponse({'detail':'Cross-origin request rejected'}, status_code=403)
|
return JSONResponse({'detail':'Cross-origin request rejected'}, status_code=403)
|
||||||
response = await call_next(request)
|
response = await call_next(request)
|
||||||
@@ -93,17 +100,18 @@ def health():
|
|||||||
storage.health()
|
storage.health()
|
||||||
except Exception:
|
except Exception:
|
||||||
raise HTTPException(503, 'Database or storage unavailable')
|
raise HTTPException(503, 'Database or storage unavailable')
|
||||||
return {'status': 'ok', 'environment': 'local', 'storage': 'minio', 'integrations': 'fake'}
|
return {'status': 'ok', 'environment': ENVIRONMENT,
|
||||||
|
'storage': 'minio' if ENVIRONMENT == 'local' else 'r2', 'integrations': 'fake'}
|
||||||
|
|
||||||
@app.get('/api/session')
|
@app.get('/api/session')
|
||||||
def session(request: Request, response: Response):
|
def session(request: Request, response: Response):
|
||||||
try:
|
try:
|
||||||
session_id = owner(request)
|
session_id = owner(request)
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
rate_limit('guest-sessions', 'local-stack', 120, 900)
|
rate_limit('guest-sessions', ENVIRONMENT, 120, 900)
|
||||||
with db.connect() as c:
|
with db.connect() as c:
|
||||||
session_id = new_session(c, response)
|
session_id = new_session(c, response)
|
||||||
return {'environment': 'local', 'cart_scope': str(session_id), 'part_bytes': PART_BYTES,
|
return {'environment': ENVIRONMENT, 'cart_scope': str(session_id), 'part_bytes': PART_BYTES,
|
||||||
'max_upload_bytes': int(os.environ.get('MAX_UPLOAD_BYTES', '5368709120'))}
|
'max_upload_bytes': int(os.environ.get('MAX_UPLOAD_BYTES', '5368709120'))}
|
||||||
|
|
||||||
@app.post('/api/freight')
|
@app.post('/api/freight')
|
||||||
@@ -126,7 +134,7 @@ def upload_row(c, upload_id, session_id, lock=False):
|
|||||||
@app.post('/api/uploads')
|
@app.post('/api/uploads')
|
||||||
def begin_upload(body: UploadStart, session_id=Depends(owner)):
|
def begin_upload(body: UploadStart, session_id=Depends(owner)):
|
||||||
if body.size > int(os.environ.get('MAX_UPLOAD_BYTES', '5368709120')):
|
if body.size > int(os.environ.get('MAX_UPLOAD_BYTES', '5368709120')):
|
||||||
raise HTTPException(413, 'File exceeds the local upload limit')
|
raise HTTPException(413, 'File exceeds the upload limit')
|
||||||
uid = uuid4()
|
uid = uuid4()
|
||||||
key = f'originals/{uid}'
|
key = f'originals/{uid}'
|
||||||
rate_limit('upload-start', str(session_id), 60, 900)
|
rate_limit('upload-start', str(session_id), 60, 900)
|
||||||
@@ -234,6 +242,8 @@ def enqueue(c, event_key, provider, payload):
|
|||||||
|
|
||||||
@app.post('/api/orders/dev-paid')
|
@app.post('/api/orders/dev-paid')
|
||||||
def dev_paid(body: Pay, session_id=Depends(owner)):
|
def dev_paid(body: Pay, session_id=Depends(owner)):
|
||||||
|
if ENVIRONMENT != 'local':
|
||||||
|
raise HTTPException(503, 'Checkout is not configured yet')
|
||||||
with db.connect() as c:
|
with db.connect() as c:
|
||||||
row = c.execute('SELECT * FROM dtf_local.quotes WHERE id=%s AND owner=%s FOR UPDATE', (body.quote_id,session_id)).fetchone()
|
row = c.execute('SELECT * FROM dtf_local.quotes WHERE id=%s AND owner=%s FOR UPDATE', (body.quote_id,session_id)).fetchone()
|
||||||
if not row:
|
if not row:
|
||||||
|
|||||||
@@ -42,7 +42,9 @@ def new_session(c, response, identity=None):
|
|||||||
sid = uuid4()
|
sid = uuid4()
|
||||||
identity = identity or uuid4()
|
identity = identity or uuid4()
|
||||||
c.execute('INSERT INTO dtf_local.sessions(id,owner) VALUES(%s,%s)', (sid,identity))
|
c.execute('INSERT INTO dtf_local.sessions(id,owner) VALUES(%s,%s)', (sid,identity))
|
||||||
response.set_cookie('dtf_session', str(sid), httponly=True, samesite='strict', max_age=86400*7)
|
response.set_cookie('dtf_session', str(sid), httponly=True,
|
||||||
|
secure=os.environ.get('COOKIE_SECURE', 'false').lower() == 'true',
|
||||||
|
samesite='strict', max_age=86400*7)
|
||||||
return identity
|
return identity
|
||||||
|
|
||||||
def transfer_guest(c, previous, identity):
|
def transfer_guest(c, previous, identity):
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ class ClamAV:
|
|||||||
|
|
||||||
def scan(self, stream, size):
|
def scan(self, stream, size):
|
||||||
if size > min(134217728, int(os.environ.get('SCAN_MAX_BYTES','134217728'))):
|
if size > min(134217728, int(os.environ.get('SCAN_MAX_BYTES','134217728'))):
|
||||||
return 'rejected', 'File exceeds the local malware scan limit'
|
return 'rejected', 'File exceeds the malware scan limit'
|
||||||
with socket.create_connection(('scanner',3310),timeout=10) as sock:
|
with socket.create_connection(('scanner',3310),timeout=10) as sock:
|
||||||
sock.settimeout(150)
|
sock.settimeout(150)
|
||||||
sock.sendall(b'zINSTREAM\0')
|
sock.sendall(b'zINSTREAM\0')
|
||||||
@@ -64,7 +64,7 @@ def scan_one(storage, scanner=None):
|
|||||||
try:state,reason=scanner.scan(stream,row['size'])
|
try:state,reason=scanner.scan(stream,row['size'])
|
||||||
finally:stream.close()
|
finally:stream.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
state,reason='error','Local malware scanner unavailable; file remains blocked'
|
state,reason='error','Malware scanner unavailable; file remains blocked'
|
||||||
c.execute("""UPDATE dtf_local.uploads SET scan_state=%s,scan_reason=%s,scanned_at=now(),
|
c.execute("""UPDATE dtf_local.uploads SET scan_state=%s,scan_reason=%s,scanned_at=now(),
|
||||||
scan_after=now()+interval '1 minute',
|
scan_after=now()+interval '1 minute',
|
||||||
expires_at=CASE WHEN %s IN ('rejected','error') THEN LEAST(expires_at,now()+interval '3 days') ELSE expires_at END
|
expires_at=CASE WHEN %s IN ('rejected','error') THEN LEAST(expires_at,now()+interval '3 days') ELSE expires_at END
|
||||||
|
|||||||
@@ -5,11 +5,11 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||||
from psycopg.types.json import Jsonb
|
from psycopg.types.json import Jsonb
|
||||||
from .adapters import FakeTiny, FakeWhatsApp, LocalS3Storage, require_local
|
from .adapters import FakeTiny, FakeWhatsApp, LocalS3Storage, require_runtime
|
||||||
from .db import connect
|
from .db import connect
|
||||||
from .scanning import ClamAV, scan_loop
|
from .scanning import ClamAV, scan_loop
|
||||||
|
|
||||||
require_local()
|
require_runtime()
|
||||||
adapters = {'tiny': FakeTiny(), 'whatsapp': FakeWhatsApp()}
|
adapters = {'tiny': FakeTiny(), 'whatsapp': FakeWhatsApp()}
|
||||||
last_tick = 0.0
|
last_tick = 0.0
|
||||||
last_cleanup = 0.0
|
last_cleanup = 0.0
|
||||||
|
|||||||
Reference in New Issue
Block a user