commit b451771169305f73e843577e8108b022e0fb983d Author: Xi Xu Date: Sat Mar 21 19:31:31 2026 +0800 chore: reset repository history diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..aac9255 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true + +[*.go] +indent_style = tab +indent_size = 4 + +[*.md] +trim_trailing_whitespace = false diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8555168 --- /dev/null +++ b/.env.example @@ -0,0 +1,26 @@ +XDROP_IMAGE=ghcr.io/xixu-me/xdrop:latest +API_ADDR=:8080 +DATABASE_URL=postgres://xdrop:xdrop@localhost:5432/xdrop?sslmode=disable +REDIS_ADDR=localhost:6379 +REDIS_PASSWORD= +REDIS_DB=0 +S3_ENDPOINT=http://localhost:9000 +S3_PUBLIC_ENDPOINT=http://localhost:8080 +S3_REGION=us-east-1 +S3_BUCKET=xdrop +S3_ACCESS_KEY=minioadmin +S3_SECRET_KEY=minioadmin +S3_USE_SSL=false +PRESIGN_TTL_SECONDS=300 +DEFAULT_EXPIRY_SECONDS=3600 +ALLOWED_ORIGINS=http://localhost:8080 +RATE_LIMIT_CREATE=20 +RATE_LIMIT_PUBLIC_READ=180 +RATE_LIMIT_DOWNLOAD_URLS=120 +CLEANUP_INTERVAL=2m +CHUNK_SIZE_BYTES=8388608 +MAX_FILE_COUNT=100 +MAX_TRANSFER_BYTES=268435456 +VITE_API_BASE_URL=/api/v1 +VITE_SITE_URL=http://localhost:8080 +VITE_ENABLE_VIDEO_METADATA_STRIP=false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f7e2def --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto eol=lf +*.sh text eol=lf diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..b3dada0 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +custom: https://xi-xu.me/#sponsorships +buy_me_a_coffee: xixu diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..0b07f8f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,80 @@ +name: Bug report +description: Report a reproducible problem in Xdrop +title: 'bug: ' +labels: + - bug +body: + - type: markdown + attributes: + value: | + Thanks for helping improve Xdrop. + + Please use this form for reproducible defects only. If you are unsure whether this is a + bug or you need help with setup, use the question form instead. + - type: checkboxes + id: preflight + attributes: + label: Preflight + options: + - label: I searched existing issues and pull requests. + required: true + - label: This is not a sensitive security report. + required: true + - label: I removed secrets, tokens, and real share links from the report. + required: true + - type: dropdown + id: area + attributes: + label: Area + options: + - Web app + - API + - Encryption or share-link flow + - Docker or local environment + - CI or automation + - Documentation + - Other + validations: + required: true + - type: textarea + id: summary + attributes: + label: What happened? + description: Describe the bug and the user-visible impact. + placeholder: Uploads resume incorrectly after refresh and the share page never reaches Ready. + validations: + required: true + - type: textarea + id: expected + attributes: + label: What did you expect to happen? + placeholder: After refresh, the upload should reconnect and eventually reach Ready. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Reproduction steps + description: Be as specific as possible. + placeholder: | + 1. Start the local Docker stack + 2. Open http://localhost:8080 + 3. Upload a large file + 4. Refresh during chunk upload + 5. Observe... + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: Browser, OS, commit/tag, and anything relevant about your setup. + placeholder: Windows 11, Chrome 134, local main branch, Docker Desktop 4.x + validations: + required: true + - type: textarea + id: logs + attributes: + label: Logs, screenshots, or extra context + description: Paste sanitized logs or add screenshots if they help. + render: shell diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..07ca272 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Support guide + url: https://github.com/xixu-me/xdrop/blob/main/SUPPORT.md + about: Use the public support process for setup help, troubleshooting, and questions. + - name: Security policy + url: https://github.com/xixu-me/xdrop/blob/main/SECURITY.md + about: Report vulnerabilities privately. Do not open public security issues. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..7046d4c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,62 @@ +name: Feature request +description: Suggest an improvement that fits Xdrop's scope +title: 'feat: ' +labels: + - enhancement +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to propose an improvement. + + Xdrop is intentionally focused on privacy-preserving file transfer. Requests that start from + the problem they solve are much easier to evaluate than implementation-first ideas. + - type: checkboxes + id: preflight + attributes: + label: Preflight + options: + - label: I searched existing issues and pull requests. + required: true + - label: This request is not a security report. + required: true + - label: I understand large scope changes may need discussion before implementation. + required: true + - type: textarea + id: problem + attributes: + label: What problem are you trying to solve? + placeholder: It is hard to tell which transfers are safe to delete after privacy mode clears local controls. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed solution + placeholder: Add a sender-side status hint explaining what metadata remains locally after privacy mode runs. + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + placeholder: Documentation update, status copy tweak, or no change. + - type: dropdown + id: area + attributes: + label: Area + options: + - Upload flow + - Receive flow + - Transfer management + - API or storage + - Local development or tooling + - Documentation + - Other + validations: + required: true + - type: textarea + id: additional + attributes: + label: Additional context + description: Mockups, related issues, constraints, or rollout concerns. diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 0000000..d6ada87 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,49 @@ +name: Question or support request +description: Ask for help with setup, usage, or expected behavior +title: 'question: ' +labels: + - question +body: + - type: markdown + attributes: + value: | + Please keep support questions public unless the topic is security-sensitive. + + If this turns out to be a bug, a maintainer can help convert it into a bug report. + - type: checkboxes + id: preflight + attributes: + label: Preflight + options: + - label: I searched existing issues and pull requests. + required: true + - label: This is not a sensitive security report. + required: true + - type: textarea + id: goal + attributes: + label: What are you trying to do? + placeholder: I want to run the full stack locally and verify the Playwright flow. + validations: + required: true + - type: textarea + id: tried + attributes: + label: What have you tried so far? + placeholder: I ran docker compose up, then bun run test:e2e, but the app never became healthy. + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: OS, browser, Docker version, Bun version, Go version, and branch if relevant. + placeholder: macOS 15, Docker 28.x, Bun 1.3.x, Go 1.26.x, branch main + validations: + required: true + - type: textarea + id: details + attributes: + label: Error output or additional context + description: Paste sanitized logs, screenshots, or command output that would help someone assist. + render: shell diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4bfe3b9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,91 @@ +version: 2 +updates: + - package-ecosystem: bun + directory: / + schedule: + interval: weekly + day: monday + time: '09:00' + timezone: Asia/Shanghai + open-pull-requests-limit: 10 + labels: + - dependencies + - javascript + reviewers: + - xixu-me + commit-message: + prefix: 'deps(bun)' + groups: + web-and-tooling: + update-types: + - minor + - patch + + - package-ecosystem: gomod + directory: /apps/api + schedule: + interval: weekly + day: monday + time: '09:30' + timezone: Asia/Shanghai + open-pull-requests-limit: 5 + labels: + - dependencies + - go + reviewers: + - xixu-me + commit-message: + prefix: 'deps(go)' + groups: + api-runtime: + update-types: + - minor + - patch + + - package-ecosystem: docker + directory: /apps/web + schedule: + interval: weekly + day: tuesday + time: '09:00' + timezone: Asia/Shanghai + labels: + - dependencies + - docker + reviewers: + - xixu-me + open-pull-requests-limit: 2 + commit-message: + prefix: 'deps(docker-web)' + + - package-ecosystem: docker + directory: /apps/api + schedule: + interval: weekly + day: tuesday + time: '09:10' + timezone: Asia/Shanghai + labels: + - dependencies + - docker + reviewers: + - xixu-me + open-pull-requests-limit: 2 + commit-message: + prefix: 'deps(docker-api)' + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: wednesday + time: '09:00' + timezone: Asia/Shanghai + labels: + - dependencies + - github-actions + reviewers: + - xixu-me + open-pull-requests-limit: 3 + commit-message: + prefix: 'deps(ci)' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..f1c7e3e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,26 @@ +## Summary + +Describe the change and the user-visible impact. + +## Linked issues + +Closes # + +## Verification + +- [ ] `bun run format:check` +- [ ] Relevant frontend checks (`bun run lint:web`, `bun run typecheck:web`, `bun run test:web`) +- [ ] Relevant backend checks (`cd apps/api && go test ./... -coverprofile=coverage.out -covermode=atomic`) +- [ ] `bun run test:e2e` for full-stack or user-flow changes + +## Checklist + +- [ ] The change fits the current project scope +- [ ] Tests were added or updated when behavior changed +- [ ] Docs were updated when setup, behavior, or contributor workflow changed +- [ ] Screenshots or recordings are attached for UI changes +- [ ] Config, Docker, API contract, or security-sensitive changes are called out below + +## Notes for reviewers + +Add anything reviewers should pay extra attention to. diff --git a/.github/workflows/auto-merge-dependencies.yml b/.github/workflows/auto-merge-dependencies.yml new file mode 100644 index 0000000..7ab0637 --- /dev/null +++ b/.github/workflows/auto-merge-dependencies.yml @@ -0,0 +1,314 @@ +name: Auto-merge Dependencies + +on: + pull_request_target: + types: + - opened + - reopened + - synchronize + - ready_for_review + workflow_run: + workflows: + - CI + types: + - completed + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + issues: write + checks: read + +jobs: + mark-safe-updates: + name: Mark safe Dependabot PRs + if: ${{ github.event_name == 'pull_request_target' && github.actor == 'dependabot[bot]' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Label updates that can auto-merge + uses: actions/github-script@v8 + env: + AUTO_MERGE_LABEL: dependencies:auto-merge + with: + script: | + const owner = context.repo.owner + const repo = context.repo.repo + const issue_number = context.issue.number + const label = process.env.AUTO_MERGE_LABEL + const pr = context.payload.pull_request + const isDraft = pr?.draft === true + const safeToMerge = !isDraft + + try { + await github.rest.issues.getLabel({ owner, repo, name: label }) + } catch (error) { + if (error.status !== 404) { + throw error + } + + await github.rest.issues.createLabel({ + owner, + repo, + name: label, + color: '0e8a16', + description: 'Dependabot updates that can merge after checks pass', + }) + } + + if (safeToMerge) { + await github.rest.issues.addLabels({ + owner, + repo, + issue_number, + labels: [label], + }) + core.info(`Added ${label} to PR #${issue_number}`) + return + } + + try { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number, + name: label, + }) + core.info(`Removed ${label} from PR #${issue_number}`) + } catch (error) { + if (error.status !== 404) { + throw error + } + core.info(`Label ${label} not present on PR #${issue_number}`) + } + + merge-when-green: + name: Merge labeled dependency PRs + if: >- + ${{ + github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.pull_requests[0] && + contains(fromJSON('["pull_request","push"]'), github.event.workflow_run.event) + }} + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Merge safe updates once all checks pass + uses: actions/github-script@v8 + env: + AUTO_MERGE_LABEL: dependencies:auto-merge + with: + script: | + const owner = context.repo.owner + const repo = context.repo.repo + const run = context.payload.workflow_run + const pullRequest = run.pull_requests?.[0] + + if (!pullRequest) { + core.info('No pull request is attached to this workflow run.') + return + } + + const prNumber = pullRequest.number + const label = process.env.AUTO_MERGE_LABEL + + const { data: pr } = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber, + }) + + if (pr.state !== 'open') { + core.info(`PR #${prNumber} is already ${pr.state}.`) + return + } + + if (pr.draft) { + core.info(`PR #${prNumber} is still a draft.`) + return + } + + if (pr.user?.login !== 'dependabot[bot]') { + core.info(`PR #${prNumber} was not opened by Dependabot.`) + return + } + + if (!pr.labels.some((item) => item.name === label)) { + core.info(`PR #${prNumber} is not marked with ${label}.`) + return + } + + if (pr.head.sha !== run.head_sha) { + core.info(`PR #${prNumber} has moved on from ${run.head_sha}; skipping stale run.`) + return + } + + const { data: checkRuns } = await github.rest.checks.listForRef({ + owner, + repo, + ref: pr.head.sha, + per_page: 100, + }) + + const unfinished = checkRuns.check_runs.filter((check) => check.status !== 'completed') + if (unfinished.length > 0) { + core.info(`PR #${prNumber} still has ${unfinished.length} check run(s) in progress.`) + return + } + + const failing = checkRuns.check_runs.filter( + (check) => !['success', 'neutral', 'skipped'].includes(check.conclusion ?? ''), + ) + + if (failing.length > 0) { + core.info( + `PR #${prNumber} has failing check runs: ${failing.map((check) => check.name).join(', ')}`, + ) + return + } + + await github.rest.pulls.merge({ + owner, + repo, + pull_number: prNumber, + sha: pr.head.sha, + merge_method: 'squash', + }) + + core.info(`Merged Dependabot PR #${prNumber}.`) + + label-and-merge-all: + name: Label and merge all open Dependabot PRs + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Label all open Dependabot PRs + uses: actions/github-script@v8 + env: + AUTO_MERGE_LABEL: dependencies:auto-merge + with: + script: | + const owner = context.repo.owner + const repo = context.repo.repo + const label = process.env.AUTO_MERGE_LABEL + + try { + await github.rest.issues.getLabel({ owner, repo, name: label }) + } catch (error) { + if (error.status !== 404) { + throw error + } + + await github.rest.issues.createLabel({ + owner, + repo, + name: label, + color: '0e8a16', + description: 'Dependabot updates that can merge after checks pass', + }) + } + + const { data: pulls } = await github.rest.pulls.list({ + owner, + repo, + state: 'open', + per_page: 100, + }) + + const dependabotPRs = pulls.filter( + (pr) => pr.user?.login === 'dependabot[bot]' && !pr.draft, + ) + + if (dependabotPRs.length === 0) { + core.info('No open non-draft Dependabot PRs found.') + return + } + + for (const pr of dependabotPRs) { + await github.rest.issues.addLabels({ + owner, + repo, + issue_number: pr.number, + labels: [label], + }) + core.info(`Added ${label} to PR #${pr.number} ("${pr.title}")`) + } + + - name: Merge labeled Dependabot PRs with passing checks + uses: actions/github-script@v8 + env: + AUTO_MERGE_LABEL: dependencies:auto-merge + with: + script: | + const owner = context.repo.owner + const repo = context.repo.repo + const label = process.env.AUTO_MERGE_LABEL + + const { data: pulls } = await github.rest.pulls.list({ + owner, + repo, + state: 'open', + per_page: 100, + }) + + const candidates = pulls.filter( + (pr) => + pr.user?.login === 'dependabot[bot]' && + !pr.draft && + pr.labels.some((l) => l.name === label), + ) + + if (candidates.length === 0) { + core.info('No labeled Dependabot PRs found to merge.') + return + } + + for (const pr of candidates) { + const { data: checkRuns } = await github.rest.checks.listForRef({ + owner, + repo, + ref: pr.head.sha, + per_page: 100, + }) + + const unfinished = checkRuns.check_runs.filter( + (check) => check.status !== 'completed', + ) + + if (unfinished.length > 0) { + core.info( + `PR #${pr.number} still has ${unfinished.length} check run(s) in progress — skipping.`, + ) + continue + } + + const failing = checkRuns.check_runs.filter( + (check) => !['success', 'neutral', 'skipped'].includes(check.conclusion ?? ''), + ) + + if (failing.length > 0) { + core.info( + `PR #${pr.number} has failing check runs: ${failing.map((c) => c.name).join(', ')} — skipping.`, + ) + continue + } + + try { + await github.rest.pulls.merge({ + owner, + repo, + pull_number: pr.number, + sha: pr.head.sha, + merge_method: 'squash', + }) + core.info(`Merged Dependabot PR #${pr.number} ("${pr.title}").`) + } catch (error) { + core.warning(`Failed to merge PR #${pr.number}: ${error.message}`) + } + } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..792b075 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,252 @@ +name: CI + +on: + pull_request: + paths-ignore: + - '**/*.md' + - '.github/ISSUE_TEMPLATE/**' + - '.github/pull_request_template.md' + - 'AGENTS.md' + - 'CODE_OF_CONDUCT.md' + - 'CONTRIBUTING.md' + - 'LICENSE' + - 'SECURITY.md' + - 'SUPPORT.md' + push: + branches: + - main + paths-ignore: + - '**/*.md' + - '.github/ISSUE_TEMPLATE/**' + - '.github/pull_request_template.md' + - 'AGENTS.md' + - 'CODE_OF_CONDUCT.md' + - 'CONTRIBUTING.md' + - 'LICENSE' + - 'SECURITY.md' + - 'SUPPORT.md' + workflow_dispatch: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + COMPOSE_FILES: -f docker-compose.yml -f docker-compose.build.yml + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + +jobs: + web: + name: Web Checks + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Check Prettier formatting + run: bun run format:check:prettier + + - name: Lint web app + run: bun run lint:web + + - name: Type-check web app + run: bun run typecheck:web + + - name: Run web unit tests with coverage + run: bun run test:web:coverage + + - name: Normalize web coverage paths for Codecov + run: | + python - <<'PY' + from pathlib import Path + + source = Path('apps/web/coverage/lcov.info') + target = Path('apps/web/coverage/lcov.codecov.info') + + normalized_lines = [] + for line in source.read_text(encoding='utf-8').splitlines(): + if line.startswith('SF:'): + path = line[3:].replace('\\', '/') + marker = '/apps/web/' + if marker in path: + path = 'apps/web/' + path.split(marker, 1)[1] + elif not path.startswith('apps/web/'): + path = 'apps/web/' + path.lstrip('./') + line = f'SF:{path}' + normalized_lines.append(line) + + target.write_text('\n'.join(normalized_lines) + '\n', encoding='utf-8') + PY + + - name: Preview web coverage source paths + shell: bash + run: | + grep '^SF:' apps/web/coverage/lcov.codecov.info | head -n 20 + + - name: Upload web coverage artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: web-coverage + path: | + apps/web/coverage/lcov.info + apps/web/coverage/lcov.codecov.info + if-no-files-found: error + + - name: Upload web coverage to Codecov + if: ${{ env.CODECOV_TOKEN != '' }} + uses: codecov/codecov-action@v5 + with: + files: apps/web/coverage/lcov.codecov.info + flags: frontend + disable_search: true + token: ${{ env.CODECOV_TOKEN }} + fail_ci_if_error: false + + - name: Build web app + run: bun run build:web + + api: + name: API Checks + runs-on: ubuntu-latest + timeout-minutes: 20 + + defaults: + run: + working-directory: apps/api + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: apps/api/go.mod + cache-dependency-path: apps/api/go.sum + + - name: Check gofmt formatting + shell: bash + run: | + unformatted="$(gofmt -l .)" + if [ -n "$unformatted" ]; then + echo "The following Go files are not gofmt-formatted:" + echo "$unformatted" + exit 1 + fi + + - name: Run API tests with coverage + run: go test ./... -coverprofile=coverage.out -covermode=atomic + + - name: Print API coverage summary + run: go tool cover -func=coverage.out + + - name: Rewrite API coverage paths for Codecov + shell: bash + run: | + sed 's#^github.com/xdrop/monorepo/#apps/api/#' coverage.out > coverage.codecov.out + + - name: Upload API coverage artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: api-coverage + path: | + apps/api/coverage.out + apps/api/coverage.codecov.out + if-no-files-found: error + + - name: Upload API coverage to Codecov + if: ${{ env.CODECOV_TOKEN != '' }} + uses: codecov/codecov-action@v5 + with: + files: apps/api/coverage.codecov.out + flags: backend + disable_search: true + token: ${{ env.CODECOV_TOKEN }} + fail_ci_if_error: false + + - name: Build API binary + run: go build -o /tmp/xdrop-api ./cmd/api + + e2e: + name: Compose Smoke And E2E + runs-on: ubuntu-latest + timeout-minutes: 35 + needs: + - web + - api + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Install Playwright browser + run: bunx playwright install --with-deps chromium + + - name: Start Docker stack + run: docker compose ${{ env.COMPOSE_FILES }} up -d --build + + - name: Wait for Xdrop endpoints + shell: bash + run: | + for _ in {1..60}; do + if curl -fsS http://localhost:8080/healthz >/dev/null && curl -fsS http://localhost:8080/ >/dev/null; then + exit 0 + fi + sleep 5 + done + echo "Xdrop endpoints did not become ready in time" >&2 + exit 1 + + - name: Run browser E2E suite + run: bun run test:e2e + + - name: Capture Docker logs + if: always() + run: docker compose ${{ env.COMPOSE_FILES }} logs --no-color > compose-logs.txt + + - name: Upload Playwright report + if: always() + uses: actions/upload-artifact@v7 + with: + name: playwright-report + path: playwright-report + if-no-files-found: ignore + + - name: Upload Playwright test results + if: always() + uses: actions/upload-artifact@v7 + with: + name: playwright-test-results + path: test-results + if-no-files-found: ignore + + - name: Upload Docker logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: compose-logs + path: compose-logs.txt + if-no-files-found: ignore + + - name: Shut down Docker stack + if: always() + run: docker compose ${{ env.COMPOSE_FILES }} down -v diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000..21a525e --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,22 @@ +name: Dependency Review + +on: + pull_request: + +permissions: + contents: read + +jobs: + dependency-review: + name: Review dependency changes + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Run dependency review + uses: actions/dependency-review-action@v4 + with: + fail-on-severity: high diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml new file mode 100644 index 0000000..6c5a275 --- /dev/null +++ b/.github/workflows/publish-images.yml @@ -0,0 +1,82 @@ +name: Publish Images + +on: + workflow_run: + workflows: + - CI + types: + - completed + branches: + - main + workflow_dispatch: + inputs: + confirm_publish: + description: 'Type publish to confirm a manual image release' + required: true + type: string + +concurrency: + group: publish-images-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + packages: write + id-token: write + +jobs: + publish: + name: Publish Xdrop + runs-on: ubuntu-latest + timeout-minutes: 30 + if: >- + ${{ + ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.confirm_publish == 'publish' + ) || + github.event.workflow_run.conclusion == 'success' + }} + + steps: + - name: Check out repository + uses: actions/checkout@v6 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ghcr.io/${{ github.repository_owner }}/xdrop + tags: | + type=raw,value=main + type=sha + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push image + uses: docker/build-push-action@v7 + with: + context: . + file: Dockerfile + push: true + platforms: linux/amd64 + build-args: | + VITE_SITE_URL=${{ vars.VITE_SITE_URL }} + VITE_API_BASE_URL=/api/v1 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: true + sbom: true diff --git a/.github/workflows/security-audits.yml b/.github/workflows/security-audits.yml new file mode 100644 index 0000000..f496d4a --- /dev/null +++ b/.github/workflows/security-audits.yml @@ -0,0 +1,201 @@ +name: Security Audits + +on: + schedule: + - cron: '43 2 * * 1' + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + bun-audit: + name: bun audit + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Run bun audit + shell: bash + run: | + set -o pipefail + bun audit --audit-level=high --json | tee bun-audit.json + + - name: Upload bun audit report + if: always() + uses: actions/upload-artifact@v7 + with: + name: bun-audit-report + path: bun-audit.json + if-no-files-found: ignore + + govulncheck: + name: govulncheck + runs-on: ubuntu-latest + timeout-minutes: 15 + + defaults: + run: + working-directory: apps/api + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: apps/api/go.mod + cache-dependency-path: apps/api/go.sum + + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: Run govulncheck + shell: bash + run: | + set -o pipefail + govulncheck ./... | tee ../../govulncheck.txt + + - name: Upload govulncheck report + if: always() + uses: actions/upload-artifact@v7 + with: + name: govulncheck-report + path: govulncheck.txt + if-no-files-found: ignore + + report-status: + name: Report security audit status + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: + - bun-audit + - govulncheck + if: always() + + steps: + - name: Create or update failure issue + if: ${{ needs.bun-audit.result != 'success' || needs.govulncheck.result != 'success' }} + uses: actions/github-script@v8 + env: + BUN_AUDIT_RESULT: ${{ needs.bun-audit.result }} + GOVULNCHECK_RESULT: ${{ needs.govulncheck.result }} + ISSUE_LABEL: security-audit + ISSUE_TITLE: Security audits failing + with: + script: | + const owner = context.repo.owner + const repo = context.repo.repo + const label = process.env.ISSUE_LABEL + const title = process.env.ISSUE_TITLE + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}` + const body = [ + 'One or more scheduled security audit jobs failed.', + '', + `- Workflow run: ${runUrl}`, + `- bun audit: ${process.env.BUN_AUDIT_RESULT}`, + `- govulncheck: ${process.env.GOVULNCHECK_RESULT}`, + `- Trigger: ${context.eventName}`, + `- Commit: ${context.sha}`, + ].join('\n') + + try { + await github.rest.issues.getLabel({ owner, repo, name: label }) + } catch (error) { + if (error.status !== 404) { + throw error + } + + await github.rest.issues.createLabel({ + owner, + repo, + name: label, + color: 'b60205', + description: 'Failures from the scheduled security audits workflow', + }) + } + + const { data: issues } = await github.rest.issues.listForRepo({ + owner, + repo, + state: 'open', + labels: label, + per_page: 100, + }) + + const existing = issues.find((issue) => issue.title === title) + + if (existing) { + await github.rest.issues.createComment({ + owner, + repo, + issue_number: existing.number, + body, + }) + core.info(`Updated existing issue #${existing.number}.`) + return + } + + const { data: created } = await github.rest.issues.create({ + owner, + repo, + title, + body, + labels: [label], + }) + + core.info(`Created issue #${created.number}.`) + + - name: Close resolved failure issue + if: ${{ needs.bun-audit.result == 'success' && needs.govulncheck.result == 'success' }} + uses: actions/github-script@v8 + env: + ISSUE_LABEL: security-audit + ISSUE_TITLE: Security audits failing + with: + script: | + const owner = context.repo.owner + const repo = context.repo.repo + const label = process.env.ISSUE_LABEL + const title = process.env.ISSUE_TITLE + + const { data: issues } = await github.rest.issues.listForRepo({ + owner, + repo, + state: 'open', + labels: label, + per_page: 100, + }) + + const existing = issues.find((issue) => issue.title === title) + if (!existing) { + core.info('No open security audit failure issue to close.') + return + } + + await github.rest.issues.createComment({ + owner, + repo, + issue_number: existing.number, + body: `Security audits recovered in ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}. Closing this alert.`, + }) + + await github.rest.issues.update({ + owner, + repo, + issue_number: existing.number, + state: 'closed', + }) + + core.info(`Closed issue #${existing.number}.`) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..93bb52d --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +node_modules/ +dist/ +coverage/ +.env +.env.* +!.env.example +*.log +.DS_Store +Thumbs.db +apps/web/dist/ +apps/web/node_modules/ +apps/api/bin/ +apps/api/.cache/ +apps/api/coverage.out +tmp/ +output/ +.playwright-cli/ +playwright-report/ +test-results/ +.codex/ diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..92a93b5 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,12 @@ +.codex +.git +coverage +dist +node_modules +output +playwright-report +test-results +tmp +apps/api/bin +apps/web/dist +apps/web/node_modules diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..10a4fd2 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,7 @@ +{ + "semi": false, + "singleQuote": true, + "printWidth": 100, + "proseWrap": "preserve", + "endOfLine": "lf" +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..43067ad --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +`Xdrop` is a monorepo with a React frontend and Go API. Put browser code in `apps/web/src` (`app`, `components`, `features`, `lib`), public assets in `apps/web/public`, shared TypeScript helpers in `packages/shared/src`, and backend code in `apps/api` with the entrypoint at `apps/api/cmd/api` and domain packages under `apps/api/internal`. End-to-end tests live in `tests/e2e`, infra files in `infra`, and repo automation in `scripts` and `.github/workflows`. + +## Build, Test, and Development Commands + +Install JS dependencies once with `bun install --frozen-lockfile`. + +- `bun run dev:web`: start the Vite dev server for the frontend. +- `bun run build:web`: type-check and build the web app, then generate SEO assets. +- `bun run lint:web`: run ESLint on `apps/web`. +- `bun run test:web` / `bun run test:web:coverage`: run Vitest, with or without coverage output. +- `bun run test:e2e`: run the Playwright suite in `tests/e2e`. +- `bun run format` / `bun run format:check`: run Prettier plus Go formatting checks. +- `go test ./... -coverprofile=coverage.out -covermode=atomic` from `apps/api`: run API tests the same way CI does. +- `go run ./cmd/api` from `apps/api`: run the API locally. +- `docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build`: boot the full stack on `http://localhost:8080`. + +## Coding Style & Naming Conventions + +Follow `.editorconfig`: 2 spaces for most files, tabs for `.go`, UTF-8, LF endings. Prettier enforces no semicolons, single quotes, and a 100-column wrap. Use PascalCase for React components, `useX` for hooks, camelCase for TS utilities, and lowercase Go package names. Keep tests beside the code they cover. + +## Testing Guidelines + +Frontend unit tests use Vitest, Testing Library, and `src/test/setup.ts`; name them `*.test.ts` or `*.test.tsx`. Browser flows use Playwright in `tests/e2e/*.spec.ts`. Backend tests use Go’s `testing` package, with integration coverage already present in `*_integration_test.go`. CI uploads frontend and backend coverage to Codecov; avoid reducing backend coverage below the current 90% target. + +## Commit & Pull Request Guidelines + +Recent history follows Conventional Commit style such as `feat:`, `fix:`, `test:`, and `docs:`. Keep commit subjects short and imperative. PRs should explain user-visible impact, link the relevant issue, and list verification steps. Include screenshots or short recordings for UI changes, and call out config, Docker, or API contract changes explicitly. + +## Security & Configuration Tips + +Start from `.env.example` for local configuration and never commit real secrets. The web dev server proxies `/api` to `localhost:8080` and `/xdrop` to MinIO on `localhost:9000`, so keep those endpoints aligned when changing local setup. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3eebb00 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,109 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and maintainers pledge to make participation in the Xdrop community a +harassment-free experience for everyone, regardless of age, body size, visible or invisible +disability, ethnicity, sex characteristics, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, race, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, +and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our community include: + +- demonstrating empathy and kindness toward other people +- being respectful of differing opinions, viewpoints, and experiences +- giving and gracefully accepting constructive feedback +- taking responsibility, apologizing to those affected by our mistakes, and learning from the + experience +- focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +- the use of sexualized language or imagery, and sexual attention or advances of any kind +- trolling, insulting or derogatory comments, and personal or political attacks +- public or private harassment +- publishing others' private information, such as a physical or email address, without their + explicit permission +- other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior +and will take appropriate and fair corrective action in response to any behavior that they deem +inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, +code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and +will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when an individual is +officially representing the community in public spaces. Examples of representing our community +include using an official email address, posting via an official social media account, or acting as +an appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project +maintainer at [hi@xi-xu.me](mailto:hi@xi-xu.me). All complaints will be reviewed and investigated +promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any +incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining the consequences for +any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community impact:** Use of inappropriate language or other behavior deemed unprofessional or +unwelcome in the community. + +**Consequence:** A private, written warning from community leaders, providing clarity around the +nature of the violation and an explanation of why the behavior was inappropriate. A public apology +may be requested. + +### 2. Warning + +**Community impact:** A violation through a single incident or series of actions. + +**Consequence:** A warning with consequences for continued behavior. No interaction with the people +involved, including unsolicited interaction with those enforcing the Code of Conduct, for a +specified period of time. This includes avoiding interactions in community spaces as well as +external channels like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community impact:** A serious violation of community standards, including sustained inappropriate +behavior. + +**Consequence:** A temporary ban from any sort of interaction or public communication with the +community for a specified period of time. No public or private interaction with the people involved, +including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this +period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community impact:** Demonstrating a pattern of violation of community standards, including +sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement +of classes of individuals. + +**Consequence:** A permanent ban from any sort of public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct/](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..fec0951 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,148 @@ +# Contributing to Xdrop + +Thank you for helping improve Xdrop. + +This repository follows the guidance from the Open Source Guides: we try to be explicit about +scope, expectations, and how to participate so contributors do not waste time on changes that are +unlikely to land. + +## Project focus + +Xdrop is focused on private file transfer with browser-side encryption. + +Changes are most likely to be accepted when they improve one or more of these areas: + +- privacy or security of the transfer flow +- reliability of uploads, downloads, expiry, or cleanup +- accessibility, usability, or performance of the current product surface +- test coverage, documentation, or developer experience +- operational hardening of the existing React + Go + Docker stack + +Changes are less likely to be accepted if they significantly broaden the project into unrelated +product areas, add accounts/social features, or increase complexity without a clear privacy or +usability win. + +## Before you start + +- Read [README.md](README.md) for the current project status and local setup. +- Follow [MESSAGING.md](MESSAGING.md) when you touch README copy, homepage copy, SEO text, or + other product-facing messaging. +- Search existing issues and pull requests before opening a new one. +- For substantial features, architecture changes, or API contract changes, open an issue first. +- For vulnerabilities or sensitive security concerns, follow [SECURITY.md](SECURITY.md) instead of + filing a public issue. +- Keep general questions and troubleshooting in public channels when possible. See + [SUPPORT.md](SUPPORT.md). + +## Development setup + +### Prerequisites + +- Bun +- Go 1.26+ +- Docker with Compose + +### Install dependencies + +```bash +bun install --frozen-lockfile +``` + +Use `.env.example` as the reference list of supported settings. Creating a local `.env` is +optional for the default dev stack; only set variables when you want to override Docker or API +defaults. + +### Start the full stack + +```bash +docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build +``` + +This brings up Xdrop, Postgres, Redis, and MinIO at `http://localhost:8080`. + +### Run the frontend only + +```bash +bun run dev:web +``` + +### Run the API only + +```bash +cd apps/api +go run ./cmd/api +``` + +## What to run before opening a PR + +Choose the checks that match your change: + +### Frontend changes + +```bash +bun run lint:web +bun run typecheck:web +bun run test:web +``` + +### Backend changes + +```bash +cd apps/api +go test ./... -coverprofile=coverage.out -covermode=atomic +``` + +### Full-stack or user-flow changes + +```bash +bun run test:e2e +``` + +### Formatting + +```bash +bun run format:check +``` + +## Coding conventions + +- Follow `.editorconfig`, Prettier, and Go formatting defaults. +- Use PascalCase for React components, `useX` for hooks, camelCase for TypeScript utilities, and + lowercase package names in Go. +- Keep tests close to the code they cover. +- Prefer focused pull requests over broad drive-by refactors. +- Add or update tests when behavior changes. + +## Pull request expectations + +Please make it easy to review your change: + +- Use a short imperative commit message, ideally in Conventional Commit style (`feat:`, `fix:`, + `docs:`, `test:`). +- Explain the user-visible impact and any API, config, Docker, or data-model changes. +- List the verification steps you ran locally. +- Include screenshots or short recordings for UI changes. +- Update documentation when behavior, setup, or contributor workflows change. + +If a change is large, split it into smaller reviewable steps whenever possible. + +## Review and maintainer expectations + +- Maintainers make the final call on whether a change fits the project vision. +- We aim to acknowledge new issues and pull requests within 7 days, but this is a part-time + project, so response times can vary. +- If you have not heard back after a week, a polite follow-up in the same thread is welcome. +- A closed issue or PR is not a judgment on the contributor. It usually means the change does not + currently fit scope, timing, or maintenance capacity. + +## Good first contributions + +If you are new to the project, good places to help include: + +- improving docs and setup clarity +- adding or tightening frontend and API test coverage +- fixing accessibility issues +- polishing error states and recovery flows +- cleaning up rough edges in local development and CI + +Thanks again for taking the time to contribute. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..11f09cf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM oven/bun:alpine AS web-build + +WORKDIR /workspace + +ARG VITE_SITE_URL="" +ARG VITE_API_BASE_URL="/api/v1" +ENV VITE_SITE_URL=${VITE_SITE_URL} +ENV VITE_API_BASE_URL=${VITE_API_BASE_URL} + +COPY package.json bun.lock tsconfig.base.json ./ +COPY packages/shared ./packages/shared +COPY apps/web ./apps/web + +RUN bun install --frozen-lockfile +RUN bun run build:web + +FROM golang:1.26-alpine AS api-build + +WORKDIR /src/apps/api + +COPY apps/api/go.mod apps/api/go.sum ./ +RUN go mod download + +COPY apps/api ./ + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/xdrop-api ./cmd/api + +FROM nginx:1.29-alpine + +COPY infra/xdrop/nginx.conf.template /etc/nginx/templates/default.conf.template +COPY infra/xdrop/entrypoint.sh /usr/local/bin/xdrop-entrypoint +COPY --from=web-build /workspace/apps/web/dist /usr/share/nginx/html +COPY --from=api-build /out/xdrop-api /usr/local/bin/xdrop-api + +RUN chmod +x /usr/local/bin/xdrop-entrypoint + +EXPOSE 80 + +ENTRYPOINT ["/usr/local/bin/xdrop-entrypoint"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e20b431 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ +GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/MESSAGING.md b/MESSAGING.md new file mode 100644 index 0000000..46e154e --- /dev/null +++ b/MESSAGING.md @@ -0,0 +1,93 @@ +# Xdrop Messaging Guide + +Use this guide to keep README copy, homepage copy, SEO metadata, Open Graph assets, package +metadata, and social profiles aligned. + +## Core Positioning + +Canonical one-liner: + +`Xdrop is an open source file transfer app that encrypts files in your browser and keeps plaintext file names, contents, and keys off the server.` + +This is the default introduction for Xdrop. If a surface only gets one sentence, use this one. + +## Canonical Copy By Surface + +README first sentence: + +`Xdrop is an open source file transfer app that encrypts files in your browser and keeps plaintext file names, contents, and keys off the server.` + +Homepage H1: + +`Encrypted file transfer.` + +Homepage body: + +`Xdrop is an open source file transfer app that encrypts files in your browser and keeps plaintext file names, contents, and keys off the server.` + +Homepage and SEO title: + +`Open Source Encrypted File Transfer in the Browser | Xdrop` + +Meta description: + +`Xdrop is an open source file transfer app that encrypts files in your browser and keeps plaintext file names, contents, and keys off the server.` + +OG card headline: + +`Open source encrypted` + +`file transfer in your browser.` + +OG card support line: + +`Plaintext file names, contents, and keys stay off the server.` + +Short social bio: + +`Open source file transfer with in-browser encryption. Plaintext file names, contents, and keys stay off the server.` + +Short technical summary: + +`Browser-encrypted file transfer with plaintext kept off the server.` + +## Messaging Priorities + +When space is limited, keep these ideas in this order: + +1. Xdrop is open source. +2. Encryption happens in the browser. +3. Plaintext file names, contents, and keys stay off the server. +4. `No account required` is a useful supporting point, but not the main definition. + +## Preferred Language + +- Prefer `encrypts files in your browser` over `browser-side encryption` in user-facing copy. +- Prefer `keeps plaintext ... off the server` over `ciphertext-only storage` unless the audience is technical. +- Prefer `open source file transfer app` when introducing Xdrop for the first time. +- Prefer `in-browser encryption` as the compact form when space is tight. +- Use `AES-256-GCM` in technical docs, threat-model explanations, and implementation notes, not as the default marketing hook. + +## Avoid + +- Avoid using `private file transfer` as the only product summary. +- Avoid mixing `private`, `secure`, `browser-side`, and `ciphertext-only` as interchangeable main taglines. +- Avoid making `no account required` the primary headline. It is a benefit, not the core definition. +- Avoid shortening the promise to just `secure uploads` because it removes the browser and server model that makes Xdrop distinct. + +## Tone + +- Clear and specific over clever. +- Technical enough to be accurate, but readable for non-specialists. +- Calm and factual instead of hype-heavy. +- Confident about the architecture, careful about broader security claims. + +## Copy Review Checklist + +Before shipping new product-facing copy, check: + +- Does it describe Xdrop as open source? +- Does it say encryption happens in the browser? +- Does it make clear that plaintext names, contents, and keys stay off the server? +- Is `no account required` used as support rather than the core identity? +- Does it avoid introducing a new summary line that conflicts with the canonical one-liner? diff --git a/README.md b/README.md new file mode 100644 index 0000000..0cfe4d8 --- /dev/null +++ b/README.md @@ -0,0 +1,414 @@ +

+ Xdrop +

+ +

+ + Codecov coverage + + + GitHub Actions CI status + + + CodeQL code scanning status + + + Container image publish status + +

+ +

+ English | 汉语 +

+ +Xdrop is an open source file transfer app that encrypts files in your browser and keeps +plaintext file names, contents, and keys off the server. + +## Highlights + +- End-to-end encryption in the browser before upload. +- Single-file and folder transfers, including local ZIP downloads for received folders. +- Resumable uploads with browser-local state for interrupted transfers. +- Expiring links, sender-side management, and optional privacy mode after upload. +- S3-compatible object storage support with PostgreSQL and Redis on the backend. + +## How It Works + +1. A sender creates a transfer in the browser. Xdrop generates a random transfer root key and a + separate link key, optionally strips removable image metadata, and prepares resumable local + state before upload begins. +2. The API creates the transfer record and returns a manage token plus upload limits. The browser + registers encrypted file metadata, then requests presigned chunk upload URLs in batches. + PostgreSQL stores transfer/file/chunk metadata, Redis enforces rate limits, and S3-compatible + storage keeps only encrypted blobs. +3. The sender shares a full link such as `/t/:transferId#k=...`. The `#k=...` fragment stays in + the browser and is used to unwrap the transfer root key locally. +4. During upload, file chunks are encrypted in a dedicated Web Worker and streamed to storage. + After every chunk is uploaded, the browser encrypts the manifest, uploads it, and finalizes the + transfer with the wrapped root key. +5. A recipient opens the link, fetches the encrypted manifest and chunk URLs, and decrypts the + transfer entirely in the browser. Folder downloads can be re-packed into a ZIP locally. +6. Background cleanup periodically removes expired or deleted transfer objects from storage. + +Xdrop keeps plaintext file names, paths, contents, and decryption keys off the server. The server +still sees operational metadata such as transfer timestamps, file counts, chunk counts, file +sizes, and rate-limit identifiers. + +Key technical details: + +- **Crypto model:** The browser generates 32-byte random secrets for the transfer root key and the + share-link key. HKDF-SHA-256 derives separate AES-256-GCM keys for the manifest and for each + file, and chunk encryption binds `transferId`, `fileId`, `chunkIndex`, size, and protocol + version as authenticated data. +- **Chunked uploads:** The server advertises chunk size, file-count, and transfer-size limits to + the browser. This repo defaults to 8 MiB chunks, up to 100 files, and a 256 MiB encrypted + transfer size cap. +- **Resume behavior:** Xdrop persists source files locally in OPFS when available and falls back + to IndexedDB-backed blobs when the staged data is still within the fallback storage limit. + Resume requests ask the API which chunks already exist so the browser only uploads missing work + after a refresh or reopen. +- **Sender controls:** The manage token is returned once on creation and stored as a SHA-256 hash + on the server. Privacy mode can scrub sender-side local controls after upload. +- **Backend responsibilities:** The API never decrypts payloads. It validates transfer state, + rate-limits endpoints, issues presigned URLs, stores metadata, and cleans up expired or deleted + objects from storage. + +## System Architecture + +```mermaid +flowchart LR + subgraph Sender["Sender browser"] + Select["Choose files or a folder"] + Worker["Crypto worker
AES-256-GCM + HKDF-SHA-256"] + Local["OPFS / IndexedDB
resume state and local controls"] + Browser["Browser app
React + upload/download runtime"] + Select --> Worker + Worker <--> Local + Browser <--> Worker + Browser <--> Local + end + + subgraph Edge["Default Xdrop deployment"] + nginx["nginx
serves SPA and proxies /api + /xdrop"] + API["Go API
transfer lifecycle, presigning, cleanup"] + nginx --> API + end + + Postgres["PostgreSQL
transfers, files, chunks, hashed manage tokens"] + Redis["Redis
rate limiting"] + Storage["S3-compatible storage
encrypted manifest and chunk objects"] + Receiver["Receiver browser
opens /t/:id#k=..."] + + Browser -->|create/register/finalize| nginx + Browser -->|presigned PUT uploads| nginx + API --> Postgres + API --> Redis + API -->|presigned PUT/GET URLs| Storage + nginx -->|/xdrop proxy| Storage + nginx -->|web app + public API| Receiver + Receiver -->|presigned GET downloads| nginx + Receiver -->|decrypts locally with #k fragment| Receiver +``` + +In the default Docker deployment, nginx serves the built frontend and proxies both `/api` and +`/xdrop`. If `S3_PUBLIC_ENDPOINT` points at a different public object-storage endpoint, presigned +upload and download requests can bypass the nginx proxy while the rest of the architecture stays +the same. + +## Deployment + +### Recommended Production Topology + +For a public deployment, run Xdrop behind a reverse proxy such as Caddy or nginx: + +- The reverse proxy terminates HTTPS for your public domain. +- The `xdrop` container listens on a loopback-only host port such as `127.0.0.1:8080`. +- MinIO should not be exposed publicly. Bind MinIO ports to `127.0.0.1` only unless you have + a specific reason to expose them. +- Set `S3_PUBLIC_ENDPOINT` and `ALLOWED_ORIGINS` to your public site URL, for example + `https://xdrop.example.com`. + +### Step 1: Get the Files + +If you only want to run the published image, you do not need to clone the whole repository on the +server. + +Download the required deployment files: + +```bash +mkdir -p xdrop/infra/minio +cd xdrop +curl -fsSL -o docker-compose.yml \ + https://github.com/xixu-me/xdrop/raw/refs/heads/main/docker-compose.yml +curl -fsSL -o infra/minio/init.sh \ + https://github.com/xixu-me/xdrop/raw/refs/heads/main/infra/minio/init.sh +chmod +x infra/minio/init.sh +``` + +Optionally, download [`.env.example`](./.env.example) as a reference for supported settings: + +```bash +curl -fsSL -o .env.example \ + https://github.com/xixu-me/xdrop/raw/refs/heads/main/.env.example +``` + +If you want to build your own image, clone the repository instead so Docker can use the full build +context. In most cases, it is better to build in CI or on a separate machine and only pull the +final image onto the server. + +### Step 2: Review Configuration + +Install Docker and Docker Compose on the server, then review the `xdrop` service environment in +`docker-compose.yml`. + +At minimum, update these values for your real deployment: + +- `S3_PUBLIC_ENDPOINT` +- `ALLOWED_ORIGINS` + +Typical production values look like this: + +```yaml +services: + minio: + ports: + - '127.0.0.1:9000:9000' + - '127.0.0.1:9001:9001' + + xdrop: + ports: + - '127.0.0.1:8080:80' + environment: + S3_PUBLIC_ENDPOINT: https://xdrop.example.com + ALLOWED_ORIGINS: https://xdrop.example.com +``` + +Treat `.env.example` as the reference list of supported settings. Changing `.env.example` alone +does not affect the running stack because the provided Compose file uses inline environment values. + +### Step 3: Use the Published Image + +```bash +docker compose up -d +``` + +This uses [`ghcr.io/xixu-me/xdrop:latest`](https://ghcr.io/xixu-me/xdrop). + +This is enough when the published image already matches the frontend settings you want. + +Important caveats: + +- Frontend build-time values such as `VITE_SITE_URL` are baked into the image. +- If your deployment uses a different public domain and you care about canonical URLs, Open Graph + metadata, JSON-LD, or sitemap generation, use your own rebuilt image instead of the published + one. + +### Step 4: Optional: Use Your Own Prebuilt Image + +```bash +XDROP_IMAGE=ghcr.io/your-org/xdrop:latest docker compose up -d +``` + +### Step 5: Optional: Build Your Own Image + +Build your own image when you need different frontend build-time settings: + +```bash +git clone https://github.com/xixu-me/xdrop.git +cd xdrop +docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build +``` + +Edit the build args in `docker-compose.build.yml` before you run that command. + +Example: + +```yaml +services: + xdrop: + build: + args: + VITE_SITE_URL: https://xdrop.example.com + VITE_API_BASE_URL: /api/v1 +``` + +On low-memory servers, building directly on the host may be slow or fail. In that case, build +elsewhere, push the image to a registry, and deploy it with `XDROP_IMAGE`. + +### Step 6: Put Xdrop Behind a Reverse Proxy + +Example `Caddyfile`: + +```caddyfile +xdrop.example.com { + encode gzip zstd + reverse_proxy 127.0.0.1:8080 +} +``` + +Then reload Caddy: + +```bash +systemctl reload caddy +``` + +After the stack starts, open . + +### Production Notes + +- The final container serves the built frontend with nginx and runs the Go API in the same + container. +- The stack includes `xdrop`, `postgres`, `redis`, `minio`, and the bucket bootstrap container. +- MinIO is intended to be private in the default single-host deployment. +- Public traffic should normally hit only the reverse proxy on ports `80` and `443`. + +## Development + +### Prerequisites + +- Bun +- Go 1.26+ +- Docker / Docker Compose + +### Step 1: Install Dependencies + +```bash +bun install --frozen-lockfile +``` + +### Step 2: Start Backing Services + +For local development, start PostgreSQL, Redis, and MinIO with Docker: + +```bash +docker compose up -d postgres redis minio minio-setup +``` + +### Step 3: Run the API + +```bash +cd apps/api +go run ./cmd/api +``` + +### Step 4: Run the Web App + +From the repo root in a second terminal: + +```bash +bun run dev:web +``` + +Open [http://localhost:5173](http://localhost:5173). During local development, the Vite dev +server proxies: + +- `/api` to `http://localhost:8080` +- `/xdrop` to `http://localhost:9000` + +This keeps frontend hot reload while talking to the local Go API and MinIO. + +## Testing + +### Web + +```bash +bun run lint:web +bun run typecheck:web +bun run test:web +bun run test:web:coverage +bun run build:web +``` + +### End-to-End + +Install Playwright browsers once if needed: + +```bash +bun run test:e2e:install +``` + +The E2E suite expects Xdrop at `http://localhost:8080` by default and uses the local `postgres` +and `redis` Compose services during the tests. Start the full stack first: + +```bash +docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build +``` + +Then run the suite: + +```bash +bun run test:e2e +``` + +Set `E2E_BASE_URL` and `E2E_API_URL` if you want to target a different environment. + +### API + +From `apps/api`: + +```bash +go test ./... -coverprofile=coverage.out -covermode=atomic +``` + +Some API integration tests use Docker-backed testcontainers. If Docker is unavailable, those tests +are skipped and coverage will be lower than CI. + +### Formatting + +```bash +bun run format +bun run format:check +``` + +## Project Structure + +```text +apps/ + api/ Go API + cmd/api/ API entrypoint + internal/ Domain packages + web/ React frontend + public/ Static assets + src/ App, components, features, and utilities +packages/ + shared/ Shared TypeScript constants and helpers + src/ Shared source files +tests/ + e2e/ Playwright end-to-end tests +infra/ Deployment and container configuration +scripts/ Repository automation and helper scripts +``` + +## Environment Variables + +See [`.env.example`](./.env.example) for the full list. The most important settings are: + +- `API_ADDR` +- `DATABASE_URL` +- `REDIS_ADDR` +- `S3_ENDPOINT` +- `S3_PUBLIC_ENDPOINT` +- `S3_BUCKET` +- `ALLOWED_ORIGINS` +- `VITE_API_BASE_URL` +- `VITE_SITE_URL` + +## License + +AGPL-3.0-only. See [`LICENSE`](./LICENSE). diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..93c8d5e --- /dev/null +++ b/README.zh.md @@ -0,0 +1,374 @@ +

+ Xdrop +

+ +

+ + Codecov coverage + + + GitHub Actions CI status + + + CodeQL code scanning status + + + Container image publish status + +

+ +

+ English | 汉语 +

+ +Xdrop 是一个开源文件传输应用，会在浏览器中先加密文件再上传，确保服务端拿不到明文文件名、文件内容和密钥。 + +## 亮点 + +- 在浏览器中完成端到端加密后再上传。 +- 支持单文件和文件夹传输，接收文件夹时也可在本地重新打包为 ZIP 下载。 +- 上传中断后可依赖浏览器本地状态继续断点续传。 +- 支持到期失效链接、发送方管理，以及上传后的可选隐私模式。 +- 后端支持兼容 S3 的对象存储，并使用 PostgreSQL 和 Redis。 + +## 工作原理 + +1. 发送方在浏览器中创建一次传输。Xdrop 会生成随机的传输根密钥和独立的链接密钥，可选地移除图片中可删除的元数据，并在上传开始前准备好可恢复的本地状态。 +2. API 创建传输记录，并返回管理令牌和上传限制。浏览器先注册加密后的文件元数据，再分批请求分块上传所需的预签名 URL。PostgreSQL 保存传输、文件和分块元数据，Redis 负责限流，兼容 S3 的对象存储只保存密文对象。 +3. 发送方分享完整链接，例如 `/t/:transferId#k=...`。其中 `#k=...` 片段只保留在浏览器端，用于在本地解开传输根密钥。 +4. 上传过程中，文件分块会在独立的 Web Worker 中加密并流式写入存储。每上传完一个分块，浏览器都会加密清单文件、上传清单，并在最后使用封装后的根密钥完成传输。 +5. 接收方打开链接后，会获取加密清单和分块 URL，并在浏览器中完成全部解密。文件夹下载时可以在本地重新打包成 ZIP。 +6. 后台清理任务会定期从存储中删除已过期或已删除的传输对象。 + +Xdrop 不会让服务端接触到明文文件名、路径、内容或解密密钥。服务端仍然能看到运行层面的元数据，例如传输时间戳、文件数量、分块数量、文件大小，以及限流标识符。 + +关键技术细节： + +- **加密模型：** 浏览器会为传输根密钥和分享链接密钥各生成一个 32 字节随机密钥。随后使用 HKDF-SHA-256 派生出清单和每个文件各自的 AES-256-GCM 密钥，并在分块加密时把 `transferId`、`fileId`、`chunkIndex`、大小和协议版本作为认证附加数据绑定进去。 +- **分块上传：** 服务端会向浏览器声明分块大小、文件数限制和传输总大小限制。当前存储库默认使用 8 MiB 分块，最多 100 个文件，加密后的传输总大小上限为 256 MiB。 +- **断点续传：** 当浏览器支持 OPFS 时，Xdrop 会把源文件持久化到本地；如果不可用，则在回退存储限制范围内使用基于 IndexedDB 的 Blob 存储。恢复上传时，浏览器会先询问 API 哪些分块已存在，因此即使刷新页面或重新打开浏览器，也只会补传缺失的部分。 +- **发送方控制：** 管理令牌只会在创建时返回一次，服务端保存的是它的 SHA-256 哈希。开启隐私模式后，发送方本地控制信息可在上传完成后被清除。 +- **后端职责：** API 从不解密载荷。它负责校验传输状态、限制速率、签发预签名 URL、存储元数据，并清理已过期或已删除的对象。 + +## 系统架构 + +```mermaid +flowchart LR + subgraph Sender["发送方浏览器"] + Select["选择文件或文件夹"] + Worker["加密 Worker
AES-256-GCM + HKDF-SHA-256"] + Local["OPFS / IndexedDB
断点状态与本地控制信息"] + Browser["浏览器应用
React + 上传/下载运行时"] + Select --> Worker + Worker <--> Local + Browser <--> Worker + Browser <--> Local + end + + subgraph Edge["默认 Xdrop 部署"] + nginx["nginx
提供 SPA 并代理 /api + /xdrop"] + API["Go API
传输生命周期、预签名、清理"] + nginx --> API + end + + Postgres["PostgreSQL
transfers、files、chunks、管理令牌哈希"] + Redis["Redis
限流"] + Storage["兼容 S3 的对象存储
加密清单与分块对象"] + Receiver["接收方浏览器
打开 /t/:id#k=..."] + + Browser -->|create/register/finalize| nginx + Browser -->|presigned PUT uploads| nginx + API --> Postgres + API --> Redis + API -->|presigned PUT/GET URLs| Storage + nginx -->|/xdrop proxy| Storage + nginx -->|web app + public API| Receiver + Receiver -->|presigned GET downloads| nginx + Receiver -->|decrypts locally with #k fragment| Receiver +``` + +在默认的 Docker 部署中，nginx 会提供构建后的前端资源，同时代理 `/api` 和 `/xdrop`。如果 `S3_PUBLIC_ENDPOINT` 指向另一个公开可访问的对象存储端点，那么预签名上传和下载请求可以绕过 nginx 代理，其余架构保持不变。 + +## 部署 + +### 推荐的生产拓扑 + +面向公网部署时，建议把 Xdrop 放在 Caddy 或 nginx 之类的反向代理后面： + +- 反向代理为你的公网域名终止 HTTPS。 +- `xdrop` 容器只监听回环地址上的主机端口，例如 `127.0.0.1:8080`。 +- 默认不建议公开暴露 MinIO。除非你有明确理由，否则应仅将 MinIO 端口绑定到 `127.0.0.1`。 +- 将 `S3_PUBLIC_ENDPOINT` 和 `ALLOWED_ORIGINS` 设置为你的公开站点 URL，例如 `https://xdrop.example.com`。 + +### 第一步：获取文件 + +如果你只想运行已经发布的镜像，就不需要在服务器上克隆整个存储库。 + +下载部署所需文件： + +```bash +mkdir -p xdrop/infra/minio +cd xdrop +curl -fsSL -o docker-compose.yml \ + https://github.com/xixu-me/xdrop/raw/refs/heads/main/docker-compose.yml +curl -fsSL -o infra/minio/init.sh \ + https://github.com/xixu-me/xdrop/raw/refs/heads/main/infra/minio/init.sh +chmod +x infra/minio/init.sh +``` + +你也可以额外下载 [`.env.example`](./.env.example) 作为支持配置项的参考： + +```bash +curl -fsSL -o .env.example \ + https://github.com/xixu-me/xdrop/raw/refs/heads/main/.env.example +``` + +如果你打算自行构建镜像，请改为克隆存储库，以便 Docker 拿到完整的构建上下文。通常更推荐在 CI 或另一台机器上完成构建，再把最终镜像拉到服务器上部署。 + +### 第二步：检查配置 + +在服务器上安装 Docker 和 Docker Compose，然后检查 `docker-compose.yml` 中 `xdrop` 服务的环境变量。 + +至少需要把这些值替换成真实部署所用配置： + +- `S3_PUBLIC_ENDPOINT` +- `ALLOWED_ORIGINS` + +典型的生产配置如下： + +```yaml +services: + minio: + ports: + - '127.0.0.1:9000:9000' + - '127.0.0.1:9001:9001' + + xdrop: + ports: + - '127.0.0.1:8080:80' + environment: + S3_PUBLIC_ENDPOINT: https://xdrop.example.com + ALLOWED_ORIGINS: https://xdrop.example.com +``` + +请把 `.env.example` 视为支持配置项的参考清单。仅修改 `.env.example` 不会影响运行中的栈，因为存储库附带的 Compose 文件使用的是内联环境变量。 + +### 第三步：使用已发布镜像 + +```bash +docker compose up -d +``` + +该命令会使用 [`ghcr.io/xixu-me/xdrop:latest`](https://ghcr.io/xixu-me/xdrop)。 + +如果已发布镜像已经符合你的前端配置需求，这一步就足够了。 + +需要注意： + +- `VITE_SITE_URL` 这类前端构建期变量会被直接烘焙进镜像。 +- 如果你的部署使用不同的公网域名，并且你在意 canonical URL、Open Graph 元数据、JSON-LD 或 sitemap 生成，请不要直接使用已发布镜像，而应自行重建镜像。 + +### 第四步：可选，使用你自己的预构建镜像 + +```bash +XDROP_IMAGE=ghcr.io/your-org/xdrop:latest docker compose up -d +``` + +### 第五步：可选，自行构建镜像 + +当你需要不同的前端构建期配置时，请自行构建镜像： + +```bash +git clone https://github.com/xixu-me/xdrop.git +cd xdrop +docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build +``` + +运行前请先修改 `docker-compose.build.yml` 中的构建参数。 + +例如： + +```yaml +services: + xdrop: + build: + args: + VITE_SITE_URL: https://xdrop.example.com + VITE_API_BASE_URL: /api/v1 +``` + +在低内存服务器上直接构建可能会很慢甚至失败。这种情况下，建议在其他机器完成构建，把镜像推送到存储库后，再通过 `XDROP_IMAGE` 部署。 + +### 第六步：把 Xdrop 放到反向代理后面 + +`Caddyfile` 示例： + +```caddyfile +xdrop.example.com { + encode gzip zstd + reverse_proxy 127.0.0.1:8080 +} +``` + +然后重载 Caddy： + +```bash +systemctl reload caddy +``` + +栈启动后，访问 。 + +### 生产环境说明 + +- 最终容器会用 nginx 提供构建后的前端，并在同一容器中运行 Go API。 +- 整个栈包含 `xdrop`、`postgres`、`redis`、`minio` 以及初始化 bucket 的容器。 +- 在默认单机部署中，MinIO 设计为私有服务。 +- 公网流量通常只应访问反向代理暴露的 `80` 和 `443` 端口。 + +## 开发 + +### 前置要求 + +- Bun +- Go 1.26+ +- Docker / Docker Compose + +### 第一步：安装依赖 + +```bash +bun install --frozen-lockfile +``` + +### 第二步：启动基础服务 + +本地开发时，用 Docker 启动 PostgreSQL、Redis 和 MinIO： + +```bash +docker compose up -d postgres redis minio minio-setup +``` + +### 第三步：运行 API + +```bash +cd apps/api +go run ./cmd/api +``` + +### 第四步：运行 Web 应用 + +在第二个终端里，从存储库根目录执行： + +```bash +bun run dev:web +``` + +打开 [http://localhost:5173](http://localhost:5173)。本地开发时，Vite 开发服务器会代理： + +- `/api` 到 `http://localhost:8080` +- `/xdrop` 到 `http://localhost:9000` + +这样既能保留前端热更新，也能连接本地 Go API 和 MinIO。 + +## 测试 + +### Web + +```bash +bun run lint:web +bun run typecheck:web +bun run test:web +bun run test:web:coverage +bun run build:web +``` + +### 端到端测试 + +如果还没安装 Playwright 浏览器，先执行一次： + +```bash +bun run test:e2e:install +``` + +E2E 套件默认会访问 `http://localhost:8080` 上的 Xdrop，并在测试期间使用本地 Compose 启动的 `postgres` 和 `redis` 服务。请先启动完整栈： + +```bash +docker compose -f docker-compose.yml -f docker-compose.build.yml up -d --build +``` + +然后运行测试： + +```bash +bun run test:e2e +``` + +如果你想指向其他环境，请设置 `E2E_BASE_URL` 和 `E2E_API_URL`。 + +### API + +在 `apps/api` 目录下执行： + +```bash +go test ./... -coverprofile=coverage.out -covermode=atomic +``` + +部分 API 集成测试会使用基于 Docker 的 testcontainers。如果 Docker 不可用，这些测试会被跳过，覆盖率也会低于 CI。 + +### 格式化 + +```bash +bun run format +bun run format:check +``` + +## 项目结构 + +```text +apps/ + api/ Go API + cmd/api/ API 入口 + internal/ 领域包 + web/ React 前端 + public/ 静态资源 + src/ 应用、组件、功能模块与工具代码 +packages/ + shared/ 共享 TypeScript 常量与辅助函数 + src/ 共享源码 +tests/ + e2e/ Playwright 端到端测试 +infra/ 部署与容器配置 +scripts/ 存储库自动化与辅助脚本 +``` + +## 环境变量 + +完整列表请查看 [`.env.example`](./.env.example)。其中最重要的设置包括： + +- `API_ADDR` +- `DATABASE_URL` +- `REDIS_ADDR` +- `S3_ENDPOINT` +- `S3_PUBLIC_ENDPOINT` +- `S3_BUCKET` +- `ALLOWED_ORIGINS` +- `VITE_API_BASE_URL` +- `VITE_SITE_URL` + +## 许可证 + +AGPL-3.0-only。详见 [`LICENSE`](./LICENSE)。 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3b8df17 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,46 @@ +# Security Policy + +## Supported versions + +Xdrop is pre-`1.0`, so security fixes are only guaranteed on the latest development line. + +| Version or branch | Supported | +| ----------------------------------------- | --------- | +| `main` | Yes | +| older commits, branches, and ad-hoc forks | No | + +## Reporting a vulnerability + +Please do not report security issues in public GitHub issues, discussions, or pull requests. + +Instead, email [hi@xi-xu.me](mailto:hi@xi-xu.me) with the subject line `Xdrop security report`. + +Include as much of the following as you can: + +- a description of the issue and the impacted component +- reproduction steps or a proof of concept +- the potential impact and any assumptions required for exploitation +- whether the issue affects confidentiality, integrity, availability, or key handling +- any suggested fix or mitigation, if you have one + +If your report involves share links, uploaded files, or secrets, sanitize them before sending. +Never post real `#k=` fragments in public places. + +## What to expect + +- We aim to acknowledge reports within 72 hours. +- We aim to provide an initial assessment within 7 days. +- We may ask for more detail, a smaller reproduction, or time to validate a fix. +- We will coordinate disclosure timing with the reporter when a report is confirmed. + +## Scope notes + +Xdrop is a file transfer system with browser-side encryption. Security-sensitive areas include: + +- key generation, wrapping, and fragment handling +- client-side encryption and decryption flows +- manifest and chunk storage behavior +- share-link lifecycle, expiry, and delete controls +- secrets, environment configuration, and deployment defaults + +Operational or configuration questions that are not vulnerabilities belong in [SUPPORT.md](SUPPORT.md). diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..435404d --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,61 @@ +# Support + +Thanks for using Xdrop. + +To keep answers discoverable and to align with open source best practices, please prefer public +support channels for non-sensitive questions. + +## Where to ask what + +### Bug reports + +Use the bug report issue form when you can reproduce a defect in the current behavior. + +Include: + +- what you expected to happen +- what actually happened +- reproduction steps +- environment details +- logs or screenshots when relevant + +### Feature requests + +Use the feature request issue form for ideas that fit Xdrop's current scope. + +For larger changes, explain the problem first. A proposal is much easier to review when it starts +with the user or maintainer pain it solves. + +### Questions and troubleshooting + +Use the question issue form for setup problems, workflow questions, or help understanding current +behavior. + +Please share enough context for someone else to help: + +- what you are trying to do +- what command or page you used +- what you already tried +- exact error output when available + +### Security issues + +Do not file public issues for vulnerabilities, exposed secrets, or anything that could put users at +risk. + +Follow [SECURITY.md](SECURITY.md) and email [hi@xi-xu.me](mailto:hi@xi-xu.me) instead. + +## Before you open an issue + +- search existing issues and pull requests first +- use the issue template that best matches your request +- keep the conversation in the issue thread instead of moving to private messages +- remove sensitive values, credentials, and full share links from logs and screenshots + +## Response expectations + +Xdrop is maintained on a best-effort basis. Maintainers will try to respond promptly, but there is +no guaranteed SLA. + +If a thread has been quiet for more than 7 days and you have new information, a polite follow-up is +welcome. diff --git a/apps/api/cmd/api/app.go b/apps/api/cmd/api/app.go new file mode 100644 index 0000000..ff8a997 --- /dev/null +++ b/apps/api/cmd/api/app.go @@ -0,0 +1,302 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + nethttp "net/http" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "github.com/xdrop/monorepo/internal/config" + apihttp "github.com/xdrop/monorepo/internal/http" + "github.com/xdrop/monorepo/internal/jobs" + "github.com/xdrop/monorepo/internal/ratelimit" + "github.com/xdrop/monorepo/internal/repo" + "github.com/xdrop/monorepo/internal/service" + "github.com/xdrop/monorepo/internal/storage" +) + +type appError struct { + message string + err error +} + +func (e *appError) Error() string { + return fmt.Sprintf("%s: %v", e.message, e.err) +} + +func (e *appError) Unwrap() error { + return e.err +} + +type dbHandle interface { + Ping(context.Context) error + Close() +} + +// redisHandle captures the small subset of Redis client behavior the runtime needs. +type redisHandle interface { + Close() error +} + +// httpServer abstracts the concrete HTTP server so tests can replace network IO. +type httpServer interface { + ListenAndServe() error + Shutdown(context.Context) error +} + +// appRuntime groups the constructed service graph and its coordinated cleanup. +type appRuntime struct { + service *service.Service + close func() +} + +// buildHooks packages side-effecting dependencies for runtime construction tests. +type buildHooks struct { + openDB func(context.Context, string) (dbHandle, error) + waitForDB func(context.Context, dbHandle) error + runMigrations func(context.Context, dbHandle) error + openStorage func(context.Context, config.Config) (storage.ObjectStorage, error) + openRedis func(config.Config) redisHandle + newLimiter func(redisHandle) ratelimit.Limiter + newRepository func(dbHandle) repo.Repository +} + +// appHooks packages top-level startup steps for CLI tests. +type appHooks struct { + loadConfig func() (config.Config, error) + buildRuntime func(context.Context, config.Config) (*appRuntime, error) + newServer func(config.Config, *slog.Logger, *service.Service) httpServer + startCleanup func(context.Context, *slog.Logger, time.Duration, *service.Service) +} + +type pgxPoolHandle struct { + *pgxpool.Pool +} + +type redisClientHandle struct { + *redis.Client +} + +// defaultBuildHooks returns the production dependency graph for runtime construction. +func defaultBuildHooks() buildHooks { + return buildHooks{ + openDB: openPostgresDB, + waitForDB: waitForPostgres, + runMigrations: runPostgresMigrations, + openStorage: openObjectStorage, + openRedis: openRedisClient, + newLimiter: newRedisLimiter, + newRepository: newPostgresRepository, + } +} + +// defaultAppHooks returns the production wiring used by the CLI entrypoint. +func defaultAppHooks() appHooks { + return appHooks{ + loadConfig: config.Load, + buildRuntime: buildDefaultRuntime, + newServer: newHTTPServer, + startCleanup: jobs.StartCleanup, + } +} + +// buildDefaultRuntime builds the production service graph with the default dependency set. +func buildDefaultRuntime(ctx context.Context, cfg config.Config) (*appRuntime, error) { + return buildRuntime(ctx, cfg, defaultBuildHooks()) +} + +func openPostgresDB(ctx context.Context, databaseURL string) (dbHandle, error) { + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return nil, err + } + return &pgxPoolHandle{Pool: pool}, nil +} + +func waitForPostgres(ctx context.Context, db dbHandle) error { + return retry(ctx, 30, 2*time.Second, func() error { + return db.Ping(ctx) + }) +} + +func runPostgresMigrations(ctx context.Context, db dbHandle) error { + handle, ok := db.(*pgxPoolHandle) + if !ok { + return fmt.Errorf("unexpected db handle type %T", db) + } + return repo.RunMigrations(ctx, handle.Pool) +} + +func openObjectStorage(ctx context.Context, cfg config.Config) (storage.ObjectStorage, error) { + return storage.NewS3Storage(ctx, storage.Config{ + Endpoint: cfg.S3Endpoint, + PublicEndpoint: cfg.S3PublicEndpoint, + Region: cfg.S3Region, + Bucket: cfg.S3Bucket, + AccessKey: cfg.S3AccessKey, + SecretKey: cfg.S3SecretKey, + UseSSL: cfg.S3UseSSL, + }) +} + +func openRedisClient(cfg config.Config) redisHandle { + return &redisClientHandle{Client: redis.NewClient(&redis.Options{ + Addr: cfg.RedisAddr, + Password: cfg.RedisPassword, + DB: cfg.RedisDB, + })} +} + +func newRedisLimiter(client redisHandle) ratelimit.Limiter { + handle, ok := client.(*redisClientHandle) + if !ok { + return nil + } + return ratelimit.NewRedisLimiter(handle.Client) +} + +func newPostgresRepository(db dbHandle) repo.Repository { + handle, ok := db.(*pgxPoolHandle) + if !ok { + return nil + } + return repo.NewPostgresRepository(handle.Pool) +} + +// buildRuntime connects the database, storage, rate limiter, and service layer. +func buildRuntime(ctx context.Context, cfg config.Config, hooks buildHooks) (*appRuntime, error) { + db, err := hooks.openDB(ctx, cfg.DatabaseURL) + if err != nil { + return nil, &appError{message: "connect postgres", err: err} + } + + closeDB := true + defer func() { + if closeDB { + db.Close() + } + }() + + waitForDB := hooks.waitForDB + if waitForDB == nil { + waitForDB = func(ctx context.Context, db dbHandle) error { + return db.Ping(ctx) + } + } + + if err := waitForDB(ctx, db); err != nil { + return nil, &appError{message: "ping postgres", err: err} + } + + if err := hooks.runMigrations(ctx, db); err != nil { + return nil, &appError{message: "run migrations", err: err} + } + + objectStorage, err := hooks.openStorage(ctx, cfg) + if err != nil { + return nil, &appError{message: "init object storage", err: err} + } + if err := objectStorage.EnsureBucket(ctx); err != nil { + return nil, &appError{message: "ensure bucket", err: err} + } + + redisClient := hooks.openRedis(cfg) + closeDB = false + + return &appRuntime{ + service: service.New(cfg, hooks.newRepository(db), objectStorage, hooks.newLimiter(redisClient)), + close: func() { + _ = redisClient.Close() + db.Close() + }, + }, nil +} + +// runMain loads configuration, builds the runtime, and blocks until the server stops. +func runMain(ctx context.Context, cancel context.CancelFunc, logger *slog.Logger, hooks appHooks) error { + cfg, err := hooks.loadConfig() + if err != nil { + return &appError{message: "load config", err: err} + } + + runtime, err := hooks.buildRuntime(ctx, cfg) + if err != nil { + return err + } + defer runtime.close() + + runServer(ctx, cancel, logger, cfg, runtime.service, hooks.newServer, hooks.startCleanup) + return nil +} + +// runServer starts background cleanup, serves HTTP traffic, and shuts down gracefully. +func runServer( + ctx context.Context, + cancel context.CancelFunc, + logger *slog.Logger, + cfg config.Config, + svc *service.Service, + newServer func(config.Config, *slog.Logger, *service.Service) httpServer, + startCleanup func(context.Context, *slog.Logger, time.Duration, *service.Service), +) { + if startCleanup != nil { + go startCleanup(ctx, logger, cfg.CleanupInterval, svc) + } + + server := newServer(cfg, logger, svc) + serverDone := make(chan error, 1) + + go func() { + logger.Info("api listening", "addr", cfg.Addr) + if err := server.ListenAndServe(); err != nil && err != nethttp.ErrServerClosed { + logger.Error("listen failed", "error", err) + cancel() + serverDone <- err + return + } + + serverDone <- nil + }() + + <-ctx.Done() + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() + if err := server.Shutdown(shutdownCtx); err != nil { + logger.Error("shutdown failed", "error", err) + } + + <-serverDone +} + +// newHTTPServer constructs the concrete net/http server used in production. +func newHTTPServer(cfg config.Config, logger *slog.Logger, svc *service.Service) httpServer { + return &nethttp.Server{ + Addr: cfg.Addr, + Handler: apihttp.NewRouter(cfg, logger, svc), + ReadHeaderTimeout: 10 * time.Second, + } +} + +// retry keeps polling a dependency until it succeeds, the context is canceled, or attempts run out. +func retry(ctx context.Context, attempts int, delay time.Duration, fn func() error) error { + var lastErr error + for attempt := 0; attempt < attempts; attempt++ { + if err := fn(); err == nil { + return nil + } else { + lastErr = err + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + } + + return lastErr +} diff --git a/apps/api/cmd/api/app_test.go b/apps/api/cmd/api/app_test.go new file mode 100644 index 0000000..eeb1205 --- /dev/null +++ b/apps/api/cmd/api/app_test.go @@ -0,0 +1,515 @@ +package main + +import ( + "context" + "errors" + "io" + "log/slog" + nethttp "net/http" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/config" + "github.com/xdrop/monorepo/internal/ratelimit" + "github.com/xdrop/monorepo/internal/repo" + "github.com/xdrop/monorepo/internal/service" + "github.com/xdrop/monorepo/internal/storage" +) + +func TestRunMainHandlesLoadConfigErrors(t *testing.T) { + t.Parallel() + + err := runMain(context.Background(), func() {}, slog.New(slog.NewTextHandler(io.Discard, nil)), appHooks{ + loadConfig: func() (config.Config, error) { + return config.Config{}, errors.New("broken config") + }, + buildRuntime: func(context.Context, config.Config) (*appRuntime, error) { + t.Fatal("buildRuntime should not be called") + return nil, nil + }, + }) + + require.ErrorContains(t, err, "load config") +} + +func TestRunMainBuildsRuntimeRunsServerAndClosesRuntime(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + cfg := config.Config{Addr: ":8080"} + server := newFakeServer() + closed := false + cleanupStarted := make(chan struct{}, 1) + + err := runMain(ctx, cancel, slog.New(slog.NewTextHandler(io.Discard, nil)), appHooks{ + loadConfig: func() (config.Config, error) { + return cfg, nil + }, + buildRuntime: func(context.Context, config.Config) (*appRuntime, error) { + return &appRuntime{ + service: nil, + close: func() { + closed = true + }, + }, nil + }, + newServer: func(config.Config, *slog.Logger, *service.Service) httpServer { + return server + }, + startCleanup: func(context.Context, *slog.Logger, time.Duration, *service.Service) { + cleanupStarted <- struct{}{} + }, + }) + + require.NoError(t, err) + require.True(t, closed) + require.True(t, server.shutdownCalled) + select { + case <-cleanupStarted: + case <-time.After(time.Second): + t.Fatal("expected cleanup loop to start") + } +} + +func TestDefaultHooksAndHelpers(t *testing.T) { + t.Parallel() + + t.Run("app error unwrap", func(t *testing.T) { + t.Parallel() + + cause := errors.New("root cause") + err := &appError{message: "outer", err: cause} + require.ErrorIs(t, err, cause) + }) + + t.Run("default hooks are wired", func(t *testing.T) { + t.Parallel() + + builder := defaultBuildHooks() + require.NotNil(t, builder.openDB) + require.NotNil(t, builder.waitForDB) + require.NotNil(t, builder.runMigrations) + require.NotNil(t, builder.openStorage) + require.NotNil(t, builder.openRedis) + require.NotNil(t, builder.newLimiter) + require.NotNil(t, builder.newRepository) + + hooks := defaultAppHooks() + require.NotNil(t, hooks.loadConfig) + require.NotNil(t, hooks.buildRuntime) + require.NotNil(t, hooks.newServer) + require.NotNil(t, hooks.startCleanup) + + require.NoError(t, waitForPostgres(context.Background(), &fakeDB{})) + require.ErrorContains(t, runPostgresMigrations(context.Background(), &fakeDB{}), "unexpected db handle type") + + pool, err := pgxpool.New(context.Background(), "postgres://xdrop:xdrop@127.0.0.1:1/xdrop?sslmode=disable&connect_timeout=1") + require.NoError(t, err) + t.Cleanup(pool.Close) + require.Error(t, runPostgresMigrations(context.Background(), &pgxPoolHandle{Pool: pool})) + + store, err := openObjectStorage(context.Background(), config.Config{ + S3Endpoint: "http://localhost:9000", + S3PublicEndpoint: "http://localhost:9000", + S3Region: "us-east-1", + S3Bucket: "xdrop", + S3AccessKey: "minioadmin", + S3SecretKey: "minioadmin", + }) + require.NoError(t, err) + require.NotNil(t, store) + + redisClient := openRedisClient(config.Config{RedisAddr: "localhost:6379"}) + require.NotNil(t, redisClient) + require.NotNil(t, newRedisLimiter(redisClient)) + require.Nil(t, newRedisLimiter(&fakeRedis{})) + + require.NotNil(t, newPostgresRepository(&pgxPoolHandle{})) + require.Nil(t, newPostgresRepository(&fakeDB{})) + + db, err := openPostgresDB(context.Background(), "postgres://xdrop:xdrop@localhost:5432/xdrop?sslmode=disable") + require.NoError(t, err) + db.Close() + + _, err = buildDefaultRuntime(context.Background(), config.Config{DatabaseURL: "postgres://%zz"}) + require.Error(t, err) + }) + + t.Run("logger and server helpers", func(t *testing.T) { + t.Parallel() + + logger := newLogger(io.Discard) + require.NotNil(t, logger) + + server := newHTTPServer(config.Config{Addr: ":8080"}, logger, nil) + httpServer, ok := server.(*nethttp.Server) + require.True(t, ok) + require.Equal(t, ":8080", httpServer.Addr) + require.Equal(t, 10*time.Second, httpServer.ReadHeaderTimeout) + require.NotNil(t, httpServer.Handler) + }) +} + +func TestBuildRuntimeHandlesFailuresAndSuccess(t *testing.T) { + t.Parallel() + + baseConfig := config.Config{ + DatabaseURL: "postgres://example.test/xdrop", + } + + t.Run("connect postgres failure", func(t *testing.T) { + t.Parallel() + + _, err := buildRuntime(context.Background(), baseConfig, buildHooks{ + openDB: func(context.Context, string) (dbHandle, error) { + return nil, errors.New("dial failed") + }, + }) + require.ErrorContains(t, err, "connect postgres") + }) + + t.Run("ping failure closes database", func(t *testing.T) { + t.Parallel() + + db := &fakeDB{pingErr: errors.New("ping failed")} + _, err := buildRuntime(context.Background(), baseConfig, buildHooks{ + openDB: func(context.Context, string) (dbHandle, error) { + return db, nil + }, + waitForDB: func(context.Context, dbHandle) error { + return db.Ping(context.Background()) + }, + }) + require.ErrorContains(t, err, "ping postgres") + require.True(t, db.closed) + }) + + t.Run("migration and storage failures close database", func(t *testing.T) { + t.Parallel() + + t.Run("migration", func(t *testing.T) { + db := &fakeDB{} + _, err := buildRuntime(context.Background(), baseConfig, buildHooks{ + openDB: func(context.Context, string) (dbHandle, error) { + return db, nil + }, + waitForDB: func(context.Context, dbHandle) error { + return nil + }, + runMigrations: func(context.Context, dbHandle) error { + return errors.New("migration failed") + }, + }) + require.ErrorContains(t, err, "run migrations") + require.True(t, db.closed) + }) + + t.Run("ensure bucket", func(t *testing.T) { + db := &fakeDB{} + store := &fakeObjectStorage{ensureBucketErr: errors.New("bucket failed")} + _, err := buildRuntime(context.Background(), baseConfig, buildHooks{ + openDB: func(context.Context, string) (dbHandle, error) { + return db, nil + }, + waitForDB: func(context.Context, dbHandle) error { + return nil + }, + runMigrations: func(context.Context, dbHandle) error { + return nil + }, + openStorage: func(context.Context, config.Config) (storage.ObjectStorage, error) { + return store, nil + }, + }) + require.ErrorContains(t, err, "ensure bucket") + require.True(t, db.closed) + require.True(t, store.ensureBucketCalled) + }) + + t.Run("open storage", func(t *testing.T) { + db := &fakeDB{} + _, err := buildRuntime(context.Background(), baseConfig, buildHooks{ + openDB: func(context.Context, string) (dbHandle, error) { + return db, nil + }, + waitForDB: func(context.Context, dbHandle) error { + return nil + }, + runMigrations: func(context.Context, dbHandle) error { + return nil + }, + openStorage: func(context.Context, config.Config) (storage.ObjectStorage, error) { + return nil, errors.New("storage failed") + }, + }) + require.ErrorContains(t, err, "init object storage") + require.True(t, db.closed) + }) + }) + + t.Run("nil waitForDB falls back to Ping", func(t *testing.T) { + t.Parallel() + + db := &fakeDB{} + redisClient := &fakeRedis{} + store := &fakeObjectStorage{} + + runtime, err := buildRuntime(context.Background(), baseConfig, buildHooks{ + openDB: func(context.Context, string) (dbHandle, error) { + return db, nil + }, + runMigrations: func(context.Context, dbHandle) error { + return nil + }, + openStorage: func(context.Context, config.Config) (storage.ObjectStorage, error) { + return store, nil + }, + openRedis: func(config.Config) redisHandle { + return redisClient + }, + newLimiter: func(redisHandle) ratelimit.Limiter { + return nil + }, + newRepository: func(dbHandle) repo.Repository { + return nil + }, + }) + require.NoError(t, err) + require.Equal(t, 1, db.pingCalls) + + runtime.close() + }) + + t.Run("success closes redis and database", func(t *testing.T) { + t.Parallel() + + db := &fakeDB{} + redisClient := &fakeRedis{} + store := &fakeObjectStorage{} + + runtime, err := buildRuntime(context.Background(), baseConfig, buildHooks{ + openDB: func(context.Context, string) (dbHandle, error) { + return db, nil + }, + waitForDB: func(context.Context, dbHandle) error { + return nil + }, + runMigrations: func(context.Context, dbHandle) error { + return nil + }, + openStorage: func(context.Context, config.Config) (storage.ObjectStorage, error) { + return store, nil + }, + openRedis: func(config.Config) redisHandle { + return redisClient + }, + newLimiter: func(redisHandle) ratelimit.Limiter { + return nil + }, + newRepository: func(dbHandle) repo.Repository { + return nil + }, + }) + require.NoError(t, err) + require.NotNil(t, runtime) + require.True(t, store.ensureBucketCalled) + require.False(t, db.closed) + require.False(t, redisClient.closed) + + runtime.close() + require.True(t, db.closed) + require.True(t, redisClient.closed) + }) +} + +func TestRunServerStartsCleanupAndShutsDownOnCancel(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + server := newFakeServer() + cleanupStarted := make(chan struct{}, 1) + done := make(chan struct{}) + + go func() { + runServer( + ctx, + cancel, + slog.New(slog.NewTextHandler(io.Discard, nil)), + config.Config{Addr: ":8080", CleanupInterval: time.Second}, + nil, + func(config.Config, *slog.Logger, *service.Service) httpServer { return server }, + func(context.Context, *slog.Logger, time.Duration, *service.Service) { cleanupStarted <- struct{}{} }, + ) + close(done) + }() + + <-server.listenStarted + cancel() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("runServer did not return after cancellation") + } + + require.True(t, server.shutdownCalled) + select { + case <-cleanupStarted: + case <-time.After(time.Second): + t.Fatal("expected cleanup loop to start") + } +} + +func TestRunServerCancelsContextWhenListenFails(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + server := &fakeServer{listenErr: errors.New("bind failed"), listenStarted: make(chan struct{})} + done := make(chan struct{}) + + go func() { + runServer( + ctx, + cancel, + slog.New(slog.NewTextHandler(io.Discard, nil)), + config.Config{Addr: ":8080"}, + nil, + func(config.Config, *slog.Logger, *service.Service) httpServer { return server }, + nil, + ) + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("runServer did not return after listen failure") + } + + require.ErrorIs(t, ctx.Err(), context.Canceled) + require.True(t, server.shutdownCalled) +} + +func TestRunServerContinuesAfterShutdownError(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + server := newFakeServer() + server.shutdownErr = errors.New("shutdown failed") + done := make(chan struct{}) + + go func() { + runServer( + ctx, + cancel, + slog.New(slog.NewTextHandler(io.Discard, nil)), + config.Config{Addr: ":8080"}, + nil, + func(config.Config, *slog.Logger, *service.Service) httpServer { return server }, + nil, + ) + close(done) + }() + + <-server.listenStarted + cancel() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("runServer did not return after shutdown failure") + } + + require.True(t, server.shutdownCalled) +} + +type fakeDB struct { + pingErr error + pingCalls int + closed bool +} + +func (d *fakeDB) Ping(context.Context) error { + d.pingCalls++ + return d.pingErr +} + +func (d *fakeDB) Close() { + d.closed = true +} + +type fakeRedis struct { + closed bool +} + +func (r *fakeRedis) Close() error { + r.closed = true + return nil +} + +type fakeObjectStorage struct { + ensureBucketErr error + ensureBucketCalled bool +} + +func (s *fakeObjectStorage) PresignUpload(context.Context, string, time.Duration) (string, error) { + return "", nil +} + +func (s *fakeObjectStorage) PresignDownload(context.Context, string, time.Duration) (string, error) { + return "", nil +} + +func (s *fakeObjectStorage) PutObject(context.Context, string, []byte, string) error { + return nil +} + +func (s *fakeObjectStorage) DeletePrefix(context.Context, string) error { + return nil +} + +func (s *fakeObjectStorage) EnsureBucket(context.Context) error { + s.ensureBucketCalled = true + return s.ensureBucketErr +} + +type fakeServer struct { + listenErr error + shutdownErr error + shutdownCalled bool + listenStarted chan struct{} + stop chan struct{} + stopClosed bool +} + +func newFakeServer() *fakeServer { + return &fakeServer{ + listenStarted: make(chan struct{}), + stop: make(chan struct{}), + } +} + +func (s *fakeServer) ListenAndServe() error { + if s.listenStarted != nil { + close(s.listenStarted) + s.listenStarted = nil + } + if s.listenErr != nil { + return s.listenErr + } + <-s.stop + return nethttp.ErrServerClosed +} + +func (s *fakeServer) Shutdown(context.Context) error { + s.shutdownCalled = true + if s.stop != nil && !s.stopClosed { + close(s.stop) + s.stopClosed = true + } + return s.shutdownErr +} diff --git a/apps/api/cmd/api/main.go b/apps/api/cmd/api/main.go new file mode 100644 index 0000000..62d8a2b --- /dev/null +++ b/apps/api/cmd/api/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "os/signal" + "syscall" +) + +var ( + // These indirections keep process-level side effects swappable in tests. + notifySignalContext = signal.NotifyContext + runCLIEntrypoint = runCLI + exitProcess = os.Exit +) + +func main() { + ctx, cancel := notifySignalContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + exitProcess(runCLIEntrypoint(ctx, cancel, newLogger(os.Stdout), defaultAppHooks())) +} + +// runCLI executes the fully wired application and converts failures into an exit code. +func runCLI(ctx context.Context, cancel context.CancelFunc, logger *slog.Logger, hooks appHooks) int { + if err := runMain(ctx, cancel, logger, hooks); err != nil { + var typedErr *appError + if errors.As(err, &typedErr) { + logger.Error(typedErr.message, "error", typedErr.err) + } else { + logger.Error("run api", "error", err) + } + return 1 + } + + return 0 +} + +// newLogger writes structured JSON logs so local and container output share one format. +func newLogger(writer io.Writer) *slog.Logger { + return slog.New(slog.NewJSONHandler(writer, &slog.HandlerOptions{Level: slog.LevelInfo})) +} diff --git a/apps/api/cmd/api/main_test.go b/apps/api/cmd/api/main_test.go new file mode 100644 index 0000000..eff112d --- /dev/null +++ b/apps/api/cmd/api/main_test.go @@ -0,0 +1,209 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "net" + nethttp "net/http" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/config" + apihttp "github.com/xdrop/monorepo/internal/http" + "github.com/xdrop/monorepo/internal/service" +) + +func TestRetryReturnsAfterSuccessfulAttempt(t *testing.T) { + t.Parallel() + + attempts := 0 + err := retry(context.Background(), 5, 5*time.Millisecond, func() error { + attempts++ + if attempts < 3 { + return errors.New("try again") + } + return nil + }) + + require.NoError(t, err) + require.Equal(t, 3, attempts) +} + +func TestRetryReturnsContextErrorWhenCancelled(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + attempts := 0 + done := make(chan struct{}) + + go func() { + time.Sleep(15 * time.Millisecond) + cancel() + close(done) + }() + + err := retry(ctx, 5, 50*time.Millisecond, func() error { + attempts++ + return errors.New("still failing") + }) + + <-done + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, attempts) +} + +func TestRetryReturnsLastErrorAfterExhaustingAttempts(t *testing.T) { + t.Parallel() + + expected := errors.New("permanent failure") + attempts := 0 + + err := retry(context.Background(), 3, 5*time.Millisecond, func() error { + attempts++ + return expected + }) + + require.ErrorIs(t, err, expected) + require.Equal(t, 3, attempts) +} + +func TestServerServesHealthzAndShutsDown(t *testing.T) { + t.Parallel() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + + server := &nethttp.Server{ + Addr: listener.Addr().String(), + Handler: apihttp.NewRouter( + config.Config{AllowedOrigins: []string{"http://localhost:5173"}}, + slog.New(slog.NewTextHandler(io.Discard, nil)), + nil, + ), + ReadHeaderTimeout: 10 * time.Second, + } + + serverErr := make(chan error, 1) + go func() { + serverErr <- server.Serve(listener) + }() + + response, err := nethttp.Get("http://" + listener.Addr().String() + "/healthz") + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, nethttp.StatusOK, response.StatusCode) + + shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, server.Shutdown(shutdownCtx)) + + err = <-serverErr + require.ErrorIs(t, err, nethttp.ErrServerClosed) +} + +func TestRunCLIReturnsExitCodesAndLogsErrors(t *testing.T) { + t.Parallel() + + t.Run("success", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + exitCode := runCLI(ctx, cancel, slog.New(slog.NewTextHandler(io.Discard, nil)), appHooks{ + loadConfig: func() (config.Config, error) { + return config.Config{}, nil + }, + buildRuntime: func(context.Context, config.Config) (*appRuntime, error) { + return &appRuntime{close: func() {}}, nil + }, + newServer: func(config.Config, *slog.Logger, *service.Service) httpServer { + return newFakeServer() + }, + }) + + require.Equal(t, 0, exitCode) + }) + + t.Run("typed application error", func(t *testing.T) { + t.Parallel() + + var buffer bytes.Buffer + exitCode := runCLI(context.Background(), func() {}, newLogger(&buffer), appHooks{ + loadConfig: func() (config.Config, error) { + return config.Config{}, errors.New("config failed") + }, + }) + + require.Equal(t, 1, exitCode) + require.Contains(t, buffer.String(), "load config") + }) + + t.Run("generic error", func(t *testing.T) { + t.Parallel() + + var buffer bytes.Buffer + exitCode := runCLI(context.Background(), func() {}, newLogger(&buffer), appHooks{ + loadConfig: func() (config.Config, error) { + return config.Config{}, nil + }, + buildRuntime: func(context.Context, config.Config) (*appRuntime, error) { + return nil, errors.New("boom") + }, + }) + + require.Equal(t, 1, exitCode) + require.Contains(t, buffer.String(), "run api") + }) +} + +func TestMainExitsWithRunCLIStatus(t *testing.T) { + t.Parallel() + + originalNotify := notifySignalContext + originalRunCLI := runCLIEntrypoint + originalExit := exitProcess + defer func() { + notifySignalContext = originalNotify + runCLIEntrypoint = originalRunCLI + exitProcess = originalExit + }() + + cancelCalled := false + exitCode := -1 + + notifySignalContext = func( + ctx context.Context, + _ ...os.Signal, + ) (context.Context, context.CancelFunc) { + return ctx, func() { + cancelCalled = true + } + } + runCLIEntrypoint = func( + ctx context.Context, + cancel context.CancelFunc, + logger *slog.Logger, + hooks appHooks, + ) int { + require.NotNil(t, ctx) + require.NotNil(t, logger) + require.NotNil(t, hooks.loadConfig) + cancel() + return 7 + } + exitProcess = func(code int) { + exitCode = code + } + + main() + + require.True(t, cancelCalled) + require.Equal(t, 7, exitCode) +} diff --git a/apps/api/go.mod b/apps/api/go.mod new file mode 100644 index 0000000..c6917cd --- /dev/null +++ b/apps/api/go.mod @@ -0,0 +1,96 @@ +module github.com/xdrop/monorepo + +go 1.26.1 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/config v1.32.12 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.8 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 + github.com/go-chi/chi/v5 v5.2.5 + github.com/go-chi/cors v1.2.2 + github.com/jackc/pgx/v5 v5.8.0 + github.com/redis/go-redis/v9 v9.18.0 + github.com/stretchr/testify v1.11.1 + github.com/testcontainers/testcontainers-go v0.41.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect + github.com/aws/smithy-go v1.24.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.2+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.10.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/shirou/gopsutil/v4 v4.26.2 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/grpc v1.79.3 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/apps/api/go.sum b/apps/api/go.sum new file mode 100644 index 0000000..51b5bd7 --- /dev/null +++ b/apps/api/go.sum @@ -0,0 +1,239 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= +github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= +github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.8 h1:nuc44j+otOY0d1e+CWwB6zul57d2YEGlgCyiq3SL0lI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.8/go.mod h1:qSFgGCN8fjdhvlLhTPZdWRWXbwfeZZWF2FEaIplYPhE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= +github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= +github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= +github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= +github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= +github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.41.0 h1:mfpsD0D36YgkxGj2LrIyxuwQ9i2wCKAD+ESsYM1wais= +github.com/testcontainers/testcontainers-go v0.41.0/go.mod h1:pdFrEIfaPl24zmBjerWTTYaY0M6UHsqA1YSvsoU40MI= +github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0 h1:AOtFXssrDlLm84A2sTTR/AhvJiYbrIuCO59d+Ro9Tb0= +github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0/go.mod h1:k2a09UKhgSp6vNpliIY0QSgm4Hi7GXVTzWvWgUemu/8= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 h1:inYW9ZhgqiDqh6BioM7DVHHzEGVq76Db5897WLGZ5Go= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0/go.mod h1:Izur+Wt8gClgMJqO/cZ8wdeeMryJ/xxiOVgFSSfpDTY= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8= +go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= +google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/apps/api/internal/config/config.go b/apps/api/internal/config/config.go new file mode 100644 index 0000000..89fdbe9 --- /dev/null +++ b/apps/api/internal/config/config.go @@ -0,0 +1,190 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Config struct { + AppEnv string + Addr string + DatabaseURL string + RedisAddr string + RedisPassword string + RedisDB int + S3Endpoint string + S3PublicEndpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseSSL bool + PresignTTL time.Duration + DefaultExpiry time.Duration + AllowedOrigins []string + CreateLimit int + PublicReadLimit int + DownloadURLLimit int + CleanupInterval time.Duration + ChunkSize int64 + MaxFileCount int + MaxTransferBytes int64 +} + +// AllowedExpiryOptions is the canonical list of transfer lifetimes accepted by the API. +var AllowedExpiryOptions = []time.Duration{ + 5 * time.Minute, + 10 * time.Minute, + 30 * time.Minute, + 1 * time.Hour, + 3 * time.Hour, + 6 * time.Hour, + 12 * time.Hour, + 24 * time.Hour, + 72 * time.Hour, + 7 * 24 * time.Hour, +} + +// Load reads environment variables, applies defaults, and validates required settings. +func Load() (Config, error) { + cfg := Config{ + AppEnv: getenv("APP_ENV", "development"), + Addr: getenv("API_ADDR", ":8080"), + DatabaseURL: getenv("DATABASE_URL", "postgres://xdrop:xdrop@localhost:5432/xdrop?sslmode=disable"), + RedisAddr: getenv("REDIS_ADDR", "localhost:6379"), + RedisPassword: getenv("REDIS_PASSWORD", ""), + S3Endpoint: getenv("S3_ENDPOINT", "http://localhost:9000"), + S3PublicEndpoint: getenv("S3_PUBLIC_ENDPOINT", "http://localhost:5173"), + S3Region: getenv("S3_REGION", "us-east-1"), + S3Bucket: getenv("S3_BUCKET", "xdrop"), + S3AccessKey: getenv("S3_ACCESS_KEY", "minioadmin"), + S3SecretKey: getenv("S3_SECRET_KEY", "minioadmin"), + S3UseSSL: getenv("S3_USE_SSL", "false") == "true", + DefaultExpiry: getenvExpiry("DEFAULT_EXPIRY_SECONDS", "DEFAULT_EXPIRY_DAYS", time.Hour), + AllowedOrigins: splitCSV(getenv("ALLOWED_ORIGINS", "http://localhost:5173,http://localhost:8080")), + CreateLimit: getenvInt("RATE_LIMIT_CREATE", 20), + PublicReadLimit: getenvInt("RATE_LIMIT_PUBLIC_READ", 180), + DownloadURLLimit: getenvInt("RATE_LIMIT_DOWNLOAD_URLS", 120), + CleanupInterval: getenvDuration("CLEANUP_INTERVAL", 2*time.Minute), + ChunkSize: getenvInt64("CHUNK_SIZE_BYTES", 8*1024*1024), + MaxFileCount: getenvInt("MAX_FILE_COUNT", 100), + MaxTransferBytes: getenvInt64("MAX_TRANSFER_BYTES", 256*1024*1024), + } + + cfg.RedisDB = getenvInt("REDIS_DB", 0) + presignSeconds := getenvInt("PRESIGN_TTL_SECONDS", 300) + cfg.PresignTTL = time.Duration(presignSeconds) * time.Second + + if cfg.DatabaseURL == "" { + return Config{}, fmt.Errorf("DATABASE_URL is required") + } + if cfg.S3Bucket == "" { + return Config{}, fmt.Errorf("S3_BUCKET is required") + } + if !IsAllowedExpiry(cfg.DefaultExpiry) { + return Config{}, fmt.Errorf("DEFAULT_EXPIRY_SECONDS must be one of the supported expiry options") + } + + return cfg, nil +} + +func getenv(key, fallback string) string { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + + return value +} + +func getenvInt(key string, fallback int) int { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + + parsed, err := strconv.Atoi(value) + if err != nil { + return fallback + } + + return parsed +} + +func getenvInt64(key string, fallback int64) int64 { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fallback + } + + return parsed +} + +func getenvDuration(key string, fallback time.Duration) time.Duration { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + + parsed, err := time.ParseDuration(value) + if err != nil { + return fallback + } + + return parsed +} + +// getenvExpiry preserves support for the previous day-based variable while preferring seconds. +func getenvExpiry(secondsKey string, legacyDaysKey string, fallback time.Duration) time.Duration { + if seconds := strings.TrimSpace(os.Getenv(secondsKey)); seconds != "" { + parsed, err := strconv.Atoi(seconds) + if err == nil && parsed > 0 { + return time.Duration(parsed) * time.Second + } + return fallback + } + + if days := strings.TrimSpace(os.Getenv(legacyDaysKey)); days != "" { + parsed, err := strconv.Atoi(days) + if err == nil && parsed > 0 { + return time.Duration(parsed) * 24 * time.Hour + } + return fallback + } + + return fallback +} + +// IsAllowedExpiry reports whether a duration matches one of the supported public options. +func IsAllowedExpiry(duration time.Duration) bool { + for _, option := range AllowedExpiryOptions { + if duration == option { + return true + } + } + + return false +} + +// splitCSV trims whitespace and drops empty values from comma-separated environment settings. +func splitCSV(value string) []string { + parts := strings.Split(value, ",") + origins := make([]string, 0, len(parts)) + + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + origins = append(origins, trimmed) + } + } + + return origins +} diff --git a/apps/api/internal/config/config_test.go b/apps/api/internal/config/config_test.go new file mode 100644 index 0000000..eaaf46d --- /dev/null +++ b/apps/api/internal/config/config_test.go @@ -0,0 +1,160 @@ +package config + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLoadUsesDefaultsWhenEnvironmentIsUnset(t *testing.T) { + clearConfigEnv(t) + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, "development", cfg.AppEnv) + require.Equal(t, ":8080", cfg.Addr) + require.Equal(t, "postgres://xdrop:xdrop@localhost:5432/xdrop?sslmode=disable", cfg.DatabaseURL) + require.Equal(t, "localhost:6379", cfg.RedisAddr) + require.Equal(t, "http://localhost:9000", cfg.S3Endpoint) + require.Equal(t, "http://localhost:5173", cfg.S3PublicEndpoint) + require.Equal(t, time.Hour, cfg.DefaultExpiry) + require.Equal(t, 5*time.Minute, cfg.PresignTTL) + require.Equal(t, []string{"http://localhost:5173", "http://localhost:8080"}, cfg.AllowedOrigins) + require.Equal(t, 20, cfg.CreateLimit) + require.Equal(t, 180, cfg.PublicReadLimit) + require.Equal(t, 120, cfg.DownloadURLLimit) + require.Equal(t, 2*time.Minute, cfg.CleanupInterval) + require.Equal(t, int64(8*1024*1024), cfg.ChunkSize) + require.Equal(t, 100, cfg.MaxFileCount) + require.Equal(t, int64(256*1024*1024), cfg.MaxTransferBytes) +} + +func TestLoadParsesCustomEnvironmentValues(t *testing.T) { + clearConfigEnv(t) + + t.Setenv("APP_ENV", "production") + t.Setenv("API_ADDR", ":9090") + t.Setenv("DATABASE_URL", "postgres://demo:demo@db:5432/demo?sslmode=disable") + t.Setenv("REDIS_ADDR", "redis:6379") + t.Setenv("REDIS_PASSWORD", "secret") + t.Setenv("REDIS_DB", "4") + t.Setenv("S3_ENDPOINT", "http://minio:9000") + t.Setenv("S3_PUBLIC_ENDPOINT", "https://files.example.test") + t.Setenv("S3_REGION", "ap-southeast-1") + t.Setenv("S3_BUCKET", "custom-bucket") + t.Setenv("S3_ACCESS_KEY", "key") + t.Setenv("S3_SECRET_KEY", "secret-key") + t.Setenv("S3_USE_SSL", "true") + t.Setenv("PRESIGN_TTL_SECONDS", "600") + t.Setenv("DEFAULT_EXPIRY_SECONDS", "10800") + t.Setenv("ALLOWED_ORIGINS", " https://app.example.test, https://admin.example.test ") + t.Setenv("RATE_LIMIT_CREATE", "7") + t.Setenv("RATE_LIMIT_PUBLIC_READ", "8") + t.Setenv("RATE_LIMIT_DOWNLOAD_URLS", "9") + t.Setenv("CLEANUP_INTERVAL", "45s") + t.Setenv("CHUNK_SIZE_BYTES", "12345") + t.Setenv("MAX_FILE_COUNT", "12") + t.Setenv("MAX_TRANSFER_BYTES", "34567") + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, "production", cfg.AppEnv) + require.Equal(t, ":9090", cfg.Addr) + require.Equal(t, "postgres://demo:demo@db:5432/demo?sslmode=disable", cfg.DatabaseURL) + require.Equal(t, "redis:6379", cfg.RedisAddr) + require.Equal(t, "secret", cfg.RedisPassword) + require.Equal(t, 4, cfg.RedisDB) + require.Equal(t, "http://minio:9000", cfg.S3Endpoint) + require.Equal(t, "https://files.example.test", cfg.S3PublicEndpoint) + require.Equal(t, "ap-southeast-1", cfg.S3Region) + require.Equal(t, "custom-bucket", cfg.S3Bucket) + require.Equal(t, "key", cfg.S3AccessKey) + require.Equal(t, "secret-key", cfg.S3SecretKey) + require.True(t, cfg.S3UseSSL) + require.Equal(t, 10*time.Minute, cfg.PresignTTL) + require.Equal(t, 3*time.Hour, cfg.DefaultExpiry) + require.Equal(t, []string{"https://app.example.test", "https://admin.example.test"}, cfg.AllowedOrigins) + require.Equal(t, 7, cfg.CreateLimit) + require.Equal(t, 8, cfg.PublicReadLimit) + require.Equal(t, 9, cfg.DownloadURLLimit) + require.Equal(t, 45*time.Second, cfg.CleanupInterval) + require.Equal(t, int64(12345), cfg.ChunkSize) + require.Equal(t, 12, cfg.MaxFileCount) + require.Equal(t, int64(34567), cfg.MaxTransferBytes) +} + +func TestLoadSupportsLegacyExpiryDays(t *testing.T) { + clearConfigEnv(t) + t.Setenv("DEFAULT_EXPIRY_DAYS", "3") + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, 72*time.Hour, cfg.DefaultExpiry) +} + +func TestLoadRejectsUnsupportedDefaultExpiry(t *testing.T) { + clearConfigEnv(t) + t.Setenv("DEFAULT_EXPIRY_SECONDS", "42") + + _, err := Load() + require.ErrorContains(t, err, "DEFAULT_EXPIRY_SECONDS must be one of the supported expiry options") +} + +func TestEnvHelpersFallBackOnInvalidValues(t *testing.T) { + t.Setenv("INT_VALUE", "not-a-number") + t.Setenv("INT64_VALUE", "nope") + t.Setenv("DURATION_VALUE", "bad-duration") + t.Setenv("EXPIRY_SECONDS", "-1") + t.Setenv("EXPIRY_DAYS", "still-bad") + + require.Equal(t, "fallback", getenv("MISSING_VALUE", "fallback")) + require.Equal(t, 12, getenvInt("INT_VALUE", 12)) + require.Equal(t, int64(34), getenvInt64("INT64_VALUE", 34)) + require.Equal(t, time.Minute, getenvDuration("DURATION_VALUE", time.Minute)) + require.Equal(t, 2*time.Hour, getenvExpiry("EXPIRY_SECONDS", "EXPIRY_DAYS", 2*time.Hour)) + + t.Setenv("EXPIRY_SECONDS", "") + require.Equal(t, 2*time.Hour, getenvExpiry("EXPIRY_SECONDS", "EXPIRY_DAYS", 2*time.Hour)) +} + +func TestSplitCSVAndAllowedExpiryHelpers(t *testing.T) { + t.Parallel() + + require.Equal(t, []string{"https://a.test", "https://b.test"}, splitCSV(" https://a.test, ,https://b.test ")) + require.True(t, IsAllowedExpiry(time.Hour)) + require.False(t, IsAllowedExpiry(2*time.Hour)) +} + +func clearConfigEnv(t *testing.T) { + t.Helper() + + for _, key := range []string{ + "APP_ENV", + "API_ADDR", + "DATABASE_URL", + "REDIS_ADDR", + "REDIS_PASSWORD", + "REDIS_DB", + "S3_ENDPOINT", + "S3_PUBLIC_ENDPOINT", + "S3_REGION", + "S3_BUCKET", + "S3_ACCESS_KEY", + "S3_SECRET_KEY", + "S3_USE_SSL", + "PRESIGN_TTL_SECONDS", + "DEFAULT_EXPIRY_SECONDS", + "DEFAULT_EXPIRY_DAYS", + "ALLOWED_ORIGINS", + "RATE_LIMIT_CREATE", + "RATE_LIMIT_PUBLIC_READ", + "RATE_LIMIT_DOWNLOAD_URLS", + "CLEANUP_INTERVAL", + "CHUNK_SIZE_BYTES", + "MAX_FILE_COUNT", + "MAX_TRANSFER_BYTES", + } { + t.Setenv(key, "") + } +} diff --git a/apps/api/internal/http/api_integration_test.go b/apps/api/internal/http/api_integration_test.go new file mode 100644 index 0000000..1559d55 --- /dev/null +++ b/apps/api/internal/http/api_integration_test.go @@ -0,0 +1,339 @@ +package http + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + nethttp "net/http" + "net/http/httptest" + "os/exec" + "runtime" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/xdrop/monorepo/internal/config" + "github.com/xdrop/monorepo/internal/ratelimit" + "github.com/xdrop/monorepo/internal/repo" + "github.com/xdrop/monorepo/internal/service" + "github.com/xdrop/monorepo/internal/storage" +) + +func TestAPITransferLifecycleEndToEnd(t *testing.T) { + skipIfDockerUnavailable(t) + + ctx := context.Background() + stack := startHTTPIntegrationStack(t, ctx) + + router := NewRouter( + stack.cfg, + slog.New(slog.NewTextHandler(io.Discard, nil)), + service.New( + stack.cfg, + repo.NewPostgresRepository(stack.db), + stack.objectStorage, + ratelimit.NewRedisLimiter(stack.redisClient), + ), + ) + server := httptest.NewServer(router) + defer server.Close() + + httpClient := server.Client() + + createResponse := struct { + TransferID string `json:"transferId"` + ManageToken string `json:"manageToken"` + }{} + doJSON(t, httpClient, nethttp.MethodPost, server.URL+"/api/v1/transfers/", "", map[string]int{ + "expiresInSeconds": 3600, + }, &createResponse) + require.NotEmpty(t, createResponse.TransferID) + require.NotEmpty(t, createResponse.ManageToken) + + doJSON(t, httpClient, nethttp.MethodPost, server.URL+"/api/v1/transfers/"+createResponse.TransferID+"/files", createResponse.ManageToken, []map[string]any{ + { + "fileId": "file-a", + "totalChunks": 1, + "ciphertextBytes": 5, + "plaintextBytes": 3, + "chunkSize": 3, + }, + }, nil) + + uploadURLs := struct { + Items []struct { + FileID string `json:"fileId"` + ChunkIndex int `json:"chunkIndex"` + ObjectKey string `json:"objectKey"` + URL string `json:"url"` + } `json:"items"` + }{} + doJSON(t, httpClient, nethttp.MethodPost, server.URL+"/api/v1/transfers/"+createResponse.TransferID+"/upload-urls", createResponse.ManageToken, map[string]any{ + "chunks": []map[string]any{{"fileId": "file-a", "chunkIndex": 0}}, + }, &uploadURLs) + require.Len(t, uploadURLs.Items, 1) + + chunkCiphertext := []byte("chunk") + uploadRequest, err := nethttp.NewRequestWithContext(ctx, nethttp.MethodPut, uploadURLs.Items[0].URL, bytes.NewReader(chunkCiphertext)) + require.NoError(t, err) + uploadRequest.Header.Set("Content-Type", "application/octet-stream") + uploadResponse, err := httpClient.Do(uploadRequest) + require.NoError(t, err) + uploadResponse.Body.Close() + require.Equal(t, nethttp.StatusOK, uploadResponse.StatusCode) + + doJSON(t, httpClient, nethttp.MethodPost, server.URL+"/api/v1/transfers/"+createResponse.TransferID+"/chunks/complete", createResponse.ManageToken, []map[string]any{ + { + "fileId": "file-a", + "chunkIndex": 0, + "ciphertextSize": len(chunkCiphertext), + "checksumSha256": "deadbeef", + }, + }, nil) + + manifestCiphertext := []byte(`{"version":1}`) + doJSON(t, httpClient, nethttp.MethodPost, server.URL+"/api/v1/transfers/"+createResponse.TransferID+"/manifest", createResponse.ManageToken, map[string]string{ + "ciphertextBase64": base64.StdEncoding.EncodeToString(manifestCiphertext), + }, nil) + + doJSON(t, httpClient, nethttp.MethodPost, server.URL+"/api/v1/transfers/"+createResponse.TransferID+"/finalize", createResponse.ManageToken, map[string]any{ + "wrappedRootKey": `{"wrapped":true}`, + "totalFiles": 1, + "totalCiphertextBytes": len(chunkCiphertext), + }, nil) + + publicTransfer := struct { + Status string `json:"status"` + WrappedRootKey string `json:"wrappedRootKey"` + ManifestURL string `json:"manifestUrl"` + ManifestCiphertextSize int64 `json:"manifestCiphertextSize"` + }{} + doJSON(t, httpClient, nethttp.MethodGet, server.URL+"/api/v1/public/transfers/"+createResponse.TransferID+"/", "", nil, &publicTransfer) + require.Equal(t, "ready", publicTransfer.Status) + require.Equal(t, `{"wrapped":true}`, publicTransfer.WrappedRootKey) + require.NotEmpty(t, publicTransfer.ManifestURL) + require.Equal(t, int64(len(manifestCiphertext)), publicTransfer.ManifestCiphertextSize) + + manifestResponse, err := httpClient.Get(publicTransfer.ManifestURL) + require.NoError(t, err) + defer manifestResponse.Body.Close() + require.Equal(t, nethttp.StatusOK, manifestResponse.StatusCode) + manifestBody, err := io.ReadAll(manifestResponse.Body) + require.NoError(t, err) + require.Equal(t, manifestCiphertext, manifestBody) + + downloadURLs := struct { + Items []struct { + FileID string `json:"fileId"` + ChunkIndex int `json:"chunkIndex"` + URL string `json:"url"` + } `json:"items"` + }{} + doJSON(t, httpClient, nethttp.MethodPost, server.URL+"/api/v1/public/transfers/"+createResponse.TransferID+"/download-urls", "", map[string]any{ + "chunks": []map[string]any{{"fileId": "file-a", "chunkIndex": 0}}, + }, &downloadURLs) + require.Len(t, downloadURLs.Items, 1) + + downloadResponse, err := httpClient.Get(downloadURLs.Items[0].URL) + require.NoError(t, err) + defer downloadResponse.Body.Close() + require.Equal(t, nethttp.StatusOK, downloadResponse.StatusCode) + downloadedChunk, err := io.ReadAll(downloadResponse.Body) + require.NoError(t, err) + require.Equal(t, chunkCiphertext, downloadedChunk) +} + +type httpIntegrationStack struct { + cfg config.Config + db *pgxpool.Pool + redisClient *redis.Client + objectStorage *storage.S3Storage +} + +func startHTTPIntegrationStack(t *testing.T, ctx context.Context) httpIntegrationStack { + t.Helper() + + db := startHTTPPostgresDB(t, ctx) + require.NoError(t, repo.RunMigrations(ctx, db)) + + redisClient := startHTTPRedisClient(t, ctx) + objectStorage, cfg := startHTTPStorage(t, ctx) + + return httpIntegrationStack{ + cfg: cfg, + db: db, + redisClient: redisClient, + objectStorage: objectStorage, + } +} + +func startHTTPPostgresDB(t *testing.T, ctx context.Context) *pgxpool.Pool { + t.Helper() + + container, err := tcpostgres.Run( + ctx, + "postgres:16-alpine", + tcpostgres.WithDatabase("xdrop"), + tcpostgres.WithUsername("xdrop"), + tcpostgres.WithPassword("xdrop"), + tcpostgres.BasicWaitStrategies(), + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, testcontainers.TerminateContainer(container)) + }) + + connectionString, err := container.ConnectionString(ctx, "sslmode=disable") + require.NoError(t, err) + + db, err := pgxpool.New(ctx, connectionString) + require.NoError(t, err) + require.NoError(t, db.Ping(ctx)) + t.Cleanup(db.Close) + + return db +} + +func startHTTPRedisClient(t *testing.T, ctx context.Context) *redis.Client { + t.Helper() + + container, err := testcontainers.Run( + ctx, + "redis:7-alpine", + testcontainers.WithExposedPorts("6379/tcp"), + testcontainers.WithWaitStrategy( + wait.ForLog("Ready to accept connections").WithStartupTimeout(60*time.Second), + ), + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, testcontainers.TerminateContainer(container)) + }) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "6379/tcp") + require.NoError(t, err) + + client := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%s", host, port.Port()), + DB: 0, + }) + require.NoError(t, client.Ping(ctx).Err()) + t.Cleanup(func() { + require.NoError(t, client.Close()) + }) + + return client +} + +func startHTTPStorage(t *testing.T, ctx context.Context) (*storage.S3Storage, config.Config) { + t.Helper() + + container, err := testcontainers.Run( + ctx, + "minio/minio:latest", + testcontainers.WithEnv(map[string]string{ + "MINIO_ROOT_USER": "minioadmin", + "MINIO_ROOT_PASSWORD": "minioadmin", + }), + testcontainers.WithExposedPorts("9000/tcp"), + testcontainers.WithCmd("server", "/data"), + testcontainers.WithWaitStrategy( + wait.ForHTTP("/minio/health/live"). + WithPort("9000/tcp"). + WithStartupTimeout(90*time.Second), + ), + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, testcontainers.TerminateContainer(container)) + }) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "9000/tcp") + require.NoError(t, err) + + endpoint := fmt.Sprintf("http://%s:%s", host, port.Port()) + objectStorage, err := storage.NewS3Storage(ctx, storage.Config{ + Endpoint: endpoint, + PublicEndpoint: endpoint, + Region: "us-east-1", + Bucket: "xdrop", + AccessKey: "minioadmin", + SecretKey: "minioadmin", + }) + require.NoError(t, err) + require.NoError(t, objectStorage.EnsureBucket(ctx)) + + cfg := config.Config{ + AllowedOrigins: []string{"http://localhost:5173"}, + ChunkSize: 8 * 1024 * 1024, + DefaultExpiry: time.Hour, + CreateLimit: 20, + PublicReadLimit: 120, + DownloadURLLimit: 120, + PresignTTL: 5 * time.Minute, + MaxFileCount: 100, + MaxTransferBytes: 256 * 1024 * 1024, + } + + return objectStorage, cfg +} + +func doJSON(t *testing.T, client *nethttp.Client, method string, url string, bearerToken string, payload any, target any) { + t.Helper() + + var body io.Reader + if payload != nil { + encoded, err := json.Marshal(payload) + require.NoError(t, err) + body = bytes.NewReader(encoded) + } + + request, err := nethttp.NewRequest(method, url, body) + require.NoError(t, err) + if payload != nil { + request.Header.Set("Content-Type", "application/json") + } + if bearerToken != "" { + request.Header.Set("Authorization", "Bearer "+bearerToken) + } + + response, err := client.Do(request) + require.NoError(t, err) + defer response.Body.Close() + + responseBody, err := io.ReadAll(response.Body) + require.NoError(t, err) + require.Less(t, response.StatusCode, 400, string(responseBody)) + + if target != nil { + require.NoError(t, json.Unmarshal(responseBody, target)) + } +} + +func skipIfDockerUnavailable(t *testing.T) { + t.Helper() + + if testing.Short() { + t.Skip("skipping docker-backed integration test in short mode") + } + if runtime.GOOS == "windows" { + t.Skip("skipping docker-backed integration test on windows") + } + + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skipf("skipping docker-backed integration test: %v", err) + } +} diff --git a/apps/api/internal/http/router.go b/apps/api/internal/http/router.go new file mode 100644 index 0000000..172df8d --- /dev/null +++ b/apps/api/internal/http/router.go @@ -0,0 +1,325 @@ +package http + +import ( + "encoding/json" + "errors" + "io" + "log/slog" + "net" + nethttp "net/http" + "strings" + "time" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/go-chi/cors" + "github.com/xdrop/monorepo/internal/config" + "github.com/xdrop/monorepo/internal/service" +) + +type Handler struct { + logger *slog.Logger + service *service.Service +} + +// NewRouter wires the public and manage APIs together with shared middleware. +func NewRouter(cfg config.Config, logger *slog.Logger, svc *service.Service) nethttp.Handler { + handler := Handler{ + logger: logger, + service: svc, + } + + router := chi.NewRouter() + router.Use(middleware.RequestID) + router.Use(middleware.RealIP) + router.Use(middleware.Recoverer) + router.Use(requestLogger(logger)) + router.Use(securityHeaders) + router.Use(cors.Handler(cors.Options{ + AllowedOrigins: cfg.AllowedOrigins, + AllowedMethods: []string{"GET", "POST", "PATCH", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Accept", "Authorization", "Content-Type"}, + ExposedHeaders: []string{"Content-Type"}, + AllowCredentials: false, + MaxAge: 300, + })) + + router.Get("/healthz", func(w nethttp.ResponseWriter, _ *nethttp.Request) { + writeJSON(w, nethttp.StatusOK, map[string]string{"status": "ok"}) + }) + + router.Route("/api/v1", func(r chi.Router) { + r.Route("/public/transfers/{transferId}", func(public chi.Router) { + public.Get("/", handler.getPublicTransfer) + public.Post("/download-urls", handler.createDownloadURLs) + }) + + r.Route("/transfers", func(manage chi.Router) { + manage.Post("/", handler.createTransfer) + manage.Route("/{transferId}", func(transfer chi.Router) { + transfer.Get("/", handler.getManageTransfer) + transfer.Patch("/", handler.patchTransfer) + transfer.Delete("/", handler.deleteTransfer) + transfer.Get("/resume", handler.resumeTransfer) + transfer.Post("/files", handler.registerFiles) + transfer.Post("/upload-urls", handler.createUploadURLs) + transfer.Post("/chunks/complete", handler.completeChunks) + transfer.Post("/manifest", handler.putManifest) + transfer.Post("/finalize", handler.finalizeTransfer) + }) + }) + }) + + return router +} + +func (h Handler) createTransfer(w nethttp.ResponseWriter, r *nethttp.Request) { + var request service.CreateTransferRequest + if err := decodeJSON(r, &request); err != nil { + writeError(w, err) + return + } + + response, err := h.service.CreateTransfer(r.Context(), clientKey(r), request) + if err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusCreated, response) +} + +func (h Handler) registerFiles(w nethttp.ResponseWriter, r *nethttp.Request) { + var request []service.RegisterFileRequest + if err := decodeJSON(r, &request); err != nil { + writeError(w, err) + return + } + + if err := h.service.RegisterFiles(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r), request); err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, map[string]bool{"ok": true}) +} + +func (h Handler) createUploadURLs(w nethttp.ResponseWriter, r *nethttp.Request) { + var request service.UploadURLRequest + if err := decodeJSON(r, &request); err != nil { + writeError(w, err) + return + } + + response, err := h.service.CreateUploadURLs(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r), request) + if err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, map[string]any{"items": response}) +} + +func (h Handler) completeChunks(w nethttp.ResponseWriter, r *nethttp.Request) { + var request []service.CompleteChunkRequest + if err := decodeJSON(r, &request); err != nil { + writeError(w, err) + return + } + + if err := h.service.CompleteChunks(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r), request); err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, map[string]bool{"ok": true}) +} + +func (h Handler) putManifest(w nethttp.ResponseWriter, r *nethttp.Request) { + var request service.ManifestUploadRequest + if err := decodeJSON(r, &request); err != nil { + writeError(w, err) + return + } + + if err := h.service.PutManifest(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r), request); err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, map[string]bool{"ok": true}) +} + +func (h Handler) finalizeTransfer(w nethttp.ResponseWriter, r *nethttp.Request) { + var request service.FinalizeTransferRequest + if err := decodeJSON(r, &request); err != nil { + writeError(w, err) + return + } + + if err := h.service.FinalizeTransfer(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r), request); err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, map[string]bool{"ok": true}) +} + +func (h Handler) getManageTransfer(w nethttp.ResponseWriter, r *nethttp.Request) { + response, err := h.service.GetManageTransfer(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r)) + if err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, response) +} + +func (h Handler) patchTransfer(w nethttp.ResponseWriter, r *nethttp.Request) { + var request service.UpdateTransferRequest + if err := decodeJSON(r, &request); err != nil { + writeError(w, err) + return + } + + if err := h.service.UpdateTransfer(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r), request); err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, map[string]bool{"ok": true}) +} + +func (h Handler) deleteTransfer(w nethttp.ResponseWriter, r *nethttp.Request) { + if err := h.service.DeleteTransfer(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r)); err != nil { + writeError(w, err) + return + } + + w.WriteHeader(nethttp.StatusNoContent) +} + +func (h Handler) resumeTransfer(w nethttp.ResponseWriter, r *nethttp.Request) { + response, err := h.service.ResumeTransfer(r.Context(), chi.URLParam(r, "transferId"), bearerToken(r)) + if err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, response) +} + +func (h Handler) getPublicTransfer(w nethttp.ResponseWriter, r *nethttp.Request) { + response, err := h.service.GetPublicTransfer(r.Context(), clientKey(r), chi.URLParam(r, "transferId")) + if err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, response) +} + +func (h Handler) createDownloadURLs(w nethttp.ResponseWriter, r *nethttp.Request) { + var request service.DownloadURLRequest + if err := decodeJSON(r, &request); err != nil { + writeError(w, err) + return + } + + response, err := h.service.CreateDownloadURLs(r.Context(), clientKey(r), chi.URLParam(r, "transferId"), request) + if err != nil { + writeError(w, err) + return + } + + writeJSON(w, nethttp.StatusOK, map[string]any{"items": response}) +} + +// decodeJSON enforces a single JSON value and rejects unknown fields for stricter contracts. +func decodeJSON(r *nethttp.Request, target any) error { + defer r.Body.Close() + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return &service.HTTPError{Status: nethttp.StatusBadRequest, Code: "invalid_json", Message: err.Error()} + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return &service.HTTPError{Status: nethttp.StatusBadRequest, Code: "invalid_json", Message: "request body must contain a single JSON value"} + } + + return nil +} + +// writeJSON serializes a response body with the expected JSON content type. +func writeJSON(w nethttp.ResponseWriter, status int, payload any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} + +// writeError translates service-layer HTTP errors into JSON API responses. +func writeError(w nethttp.ResponseWriter, err error) { + var httpErr *service.HTTPError + if errors.As(err, &httpErr) { + writeJSON(w, httpErr.Status, map[string]string{ + "error": httpErr.Code, + "message": httpErr.Message, + }) + return + } + + writeJSON(w, nethttp.StatusInternalServerError, map[string]string{ + "error": "internal_error", + "message": "internal server error", + }) +} + +func bearerToken(r *nethttp.Request) string { + value := strings.TrimSpace(r.Header.Get("Authorization")) + if !strings.HasPrefix(strings.ToLower(value), "bearer ") { + return "" + } + + return strings.TrimSpace(value[7:]) +} + +func clientKey(r *nethttp.Request) string { + value := strings.TrimSpace(r.RemoteAddr) + if value == "" { + return "" + } + host, _, err := net.SplitHostPort(value) + if err == nil { + return strings.TrimSpace(host) + } + + return value +} + +// securityHeaders adds a baseline set of defensive response headers to every request. +func securityHeaders(next nethttp.Handler) nethttp.Handler { + return nethttp.HandlerFunc(func(w nethttp.ResponseWriter, r *nethttp.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Referrer-Policy", "same-origin") + w.Header().Set("X-Frame-Options", "DENY") + w.Header().Set("Cross-Origin-Opener-Policy", "same-origin") + w.Header().Set("Cross-Origin-Resource-Policy", "same-site") + next.ServeHTTP(w, r) + }) +} + +// requestLogger records lightweight request metadata without buffering response bodies. +func requestLogger(logger *slog.Logger) func(nethttp.Handler) nethttp.Handler { + return func(next nethttp.Handler) nethttp.Handler { + return nethttp.HandlerFunc(func(w nethttp.ResponseWriter, r *nethttp.Request) { + startedAt := time.Now() + next.ServeHTTP(w, r) + logger.Info("request complete", + "method", r.Method, + "path", r.URL.Path, + "remote_addr", r.RemoteAddr, + "duration_ms", time.Since(startedAt).Milliseconds(), + ) + }) + } +} diff --git a/apps/api/internal/http/router_test.go b/apps/api/internal/http/router_test.go new file mode 100644 index 0000000..3028f14 --- /dev/null +++ b/apps/api/internal/http/router_test.go @@ -0,0 +1,689 @@ +package http + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "io" + "log/slog" + nethttp "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/config" + "github.com/xdrop/monorepo/internal/models" + "github.com/xdrop/monorepo/internal/ratelimit" + "github.com/xdrop/monorepo/internal/repo" + "github.com/xdrop/monorepo/internal/service" +) + +func TestCreateTransferRateLimitUsesIPAddressWithoutPort(t *testing.T) { + t.Parallel() + + cfg := testRouterConfig() + cfg.CreateLimit = 1 + + router := NewRouter( + cfg, + slog.New(slog.NewTextHandler(io.Discard, nil)), + service.New(cfg, newRouterRepository(), &routerStorage{}, ratelimit.NewMemoryLimiter()), + ) + + first := performJSONRequest(t, router, nethttp.MethodPost, "/api/v1/transfers/", `{"expiresInSeconds":3600}`, "198.51.100.42:40001") + require.Equal(t, nethttp.StatusCreated, first.Code) + + second := performJSONRequest(t, router, nethttp.MethodPost, "/api/v1/transfers/", `{"expiresInSeconds":3600}`, "198.51.100.42:40002") + require.Equal(t, nethttp.StatusTooManyRequests, second.Code) + require.Contains(t, second.Body.String(), `"error":"rate_limited"`) +} + +func TestCreateTransferRejectsTrailingJSONPayload(t *testing.T) { + t.Parallel() + + router := newTestRouter() + response := performJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/transfers/", + `{"expiresInSeconds":3600}{"ignored":true}`, + "198.51.100.42:40001", + ) + + require.Equal(t, nethttp.StatusBadRequest, response.Code) + require.Contains(t, response.Body.String(), `"error":"invalid_json"`) +} + +func TestCreateTransferRejectsUnknownFields(t *testing.T) { + t.Parallel() + + router := newTestRouter() + response := performJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/transfers/", + `{"expiresInSeconds":3600,"extra":true}`, + "198.51.100.42:40001", + ) + + require.Equal(t, nethttp.StatusBadRequest, response.Code) + require.Contains(t, response.Body.String(), `"error":"invalid_json"`) +} + +func TestHealthzAppliesSecurityHeaders(t *testing.T) { + t.Parallel() + + router := newTestRouter() + request := httptest.NewRequest(nethttp.MethodGet, "/healthz", nil) + response := httptest.NewRecorder() + + router.ServeHTTP(response, request) + + require.Equal(t, nethttp.StatusOK, response.Code) + require.Equal(t, "nosniff", response.Header().Get("X-Content-Type-Options")) + require.Equal(t, "DENY", response.Header().Get("X-Frame-Options")) + require.Equal(t, "same-origin", response.Header().Get("Cross-Origin-Opener-Policy")) +} + +func newTestRouter() nethttp.Handler { + cfg := testRouterConfig() + return NewRouter( + cfg, + slog.New(slog.NewTextHandler(io.Discard, nil)), + service.New(cfg, newRouterRepository(), &routerStorage{}, ratelimit.NewMemoryLimiter()), + ) +} + +func testRouterConfig() config.Config { + return config.Config{ + AllowedOrigins: []string{"http://localhost:5173"}, + ChunkSize: 8 * 1024 * 1024, + DefaultExpiry: time.Hour, + CreateLimit: 20, + PublicReadLimit: 120, + DownloadURLLimit: 120, + PresignTTL: 5 * time.Minute, + MaxFileCount: 100, + MaxTransferBytes: 256 * 1024 * 1024, + } +} + +func performJSONRequest(t *testing.T, handler nethttp.Handler, method string, path string, body string, remoteAddr string) *httptest.ResponseRecorder { + t.Helper() + + request := httptest.NewRequest(method, path, bytes.NewBufferString(body)) + request.Header.Set("Content-Type", "application/json") + request.RemoteAddr = remoteAddr + + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + return response +} + +type routerRepository struct { + transfers map[string]models.Transfer + files map[string][]models.TransferFile + chunks map[string][]models.TransferChunk +} + +func newRouterRepository() *routerRepository { + return &routerRepository{ + transfers: map[string]models.Transfer{}, + files: map[string][]models.TransferFile{}, + chunks: map[string][]models.TransferChunk{}, + } +} + +func (r *routerRepository) CreateTransfer(_ context.Context, transfer models.Transfer) error { + r.transfers[transfer.ID] = transfer + return nil +} + +func (r *routerRepository) GetTransfer(_ context.Context, transferID string) (models.Transfer, error) { + transfer, ok := r.transfers[transferID] + if !ok { + return models.Transfer{}, repo.ErrNotFound + } + return transfer, nil +} + +func (r *routerRepository) RegisterFiles(_ context.Context, transferID string, files []models.TransferFile) error { + r.files[transferID] = append([]models.TransferFile{}, files...) + transfer := r.transfers[transferID] + transfer.Status = models.TransferStatusUploading + r.transfers[transferID] = transfer + return nil +} + +func (r *routerRepository) ListFiles(_ context.Context, transferID string) ([]models.TransferFile, error) { + return append([]models.TransferFile{}, r.files[transferID]...), nil +} + +func (r *routerRepository) CompleteChunks(_ context.Context, transferID string, chunks []models.TransferChunk) error { + r.chunks[transferID] = append(r.chunks[transferID], chunks...) + files := r.files[transferID] + for fileIndex := range files { + uploaded := 0 + for _, chunk := range r.chunks[transferID] { + if chunk.OpaqueFileID == files[fileIndex].OpaqueFileID { + uploaded++ + } + } + if uploaded >= files[fileIndex].TotalChunks { + files[fileIndex].UploadStatus = "complete" + } + } + r.files[transferID] = files + return nil +} + +func (r *routerRepository) GetResumeState(ctx context.Context, transferID string) (models.TransferResumeState, error) { + transfer, err := r.GetTransfer(ctx, transferID) + if err != nil { + return models.TransferResumeState{}, err + } + + uploaded := map[string][]int{} + for _, chunk := range r.chunks[transferID] { + uploaded[chunk.OpaqueFileID] = append(uploaded[chunk.OpaqueFileID], chunk.ChunkIndex) + } + + return models.TransferResumeState{ + Transfer: transfer, + Files: append([]models.TransferFile{}, r.files[transferID]...), + UploadedChunks: uploaded, + }, nil +} + +func (r *routerRepository) SetManifest(_ context.Context, transferID string, objectKey string, ciphertextSize int64) error { + transfer := r.transfers[transferID] + transfer.ManifestObjectKey = objectKey + transfer.ManifestCiphertextSize = ciphertextSize + r.transfers[transferID] = transfer + return nil +} + +func (r *routerRepository) FinalizeTransfer(_ context.Context, transferID string, wrappedRootKey string, totalFiles int, totalCiphertextBytes int64) error { + transfer := r.transfers[transferID] + transfer.Status = models.TransferStatusReady + transfer.WrappedRootKey = wrappedRootKey + transfer.TotalFiles = totalFiles + transfer.TotalCiphertextBytes = totalCiphertextBytes + now := time.Now().UTC() + transfer.FinalizedAt = &now + r.transfers[transferID] = transfer + return nil +} + +func (r *routerRepository) UpdateTransfer(_ context.Context, transferID string, params models.UpdateTransferParams) error { + transfer := r.transfers[transferID] + if params.ManifestObjectKey != nil { + transfer.ManifestObjectKey = *params.ManifestObjectKey + } + if params.ExpiresAt != nil { + transfer.ExpiresAt = *params.ExpiresAt + } + if params.ManifestCiphertextSize != nil { + transfer.ManifestCiphertextSize = *params.ManifestCiphertextSize + } + r.transfers[transferID] = transfer + return nil +} + +func (r *routerRepository) MarkDeleted(_ context.Context, transferID string) error { + transfer := r.transfers[transferID] + now := time.Now().UTC() + transfer.Status = models.TransferStatusDeleted + transfer.DeletedAt = &now + r.transfers[transferID] = transfer + return nil +} + +func (r *routerRepository) ListCleanupCandidates(context.Context, int) ([]models.Transfer, error) { + return nil, nil +} + +func (r *routerRepository) MarkPurged(context.Context, string) error { + return nil +} + +type routerStorage struct{} + +func (s *routerStorage) PresignUpload(_ context.Context, objectKey string, _ time.Duration) (string, error) { + return "https://example.test/upload/" + objectKey, nil +} + +func (s *routerStorage) PresignDownload(_ context.Context, objectKey string, _ time.Duration) (string, error) { + return "https://example.test/download/" + objectKey, nil +} + +func (s *routerStorage) PutObject(context.Context, string, []byte, string) error { + return nil +} + +func (s *routerStorage) DeletePrefix(context.Context, string) error { + return nil +} + +func (s *routerStorage) EnsureBucket(context.Context) error { + return nil +} + +func TestManageTransferEndpointsLifecycle(t *testing.T) { + t.Parallel() + + router, repository := newTestRouterWithRepository() + + createResponse := struct { + TransferID string `json:"transferId"` + ManageToken string `json:"manageToken"` + }{} + create := performJSONRequest(t, router, nethttp.MethodPost, "/api/v1/transfers/", `{"expiresInSeconds":3600}`, "198.51.100.42:40001") + require.Equal(t, nethttp.StatusCreated, create.Code) + require.NoError(t, json.Unmarshal(create.Body.Bytes(), &createResponse)) + + getResponse := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodGet, + "/api/v1/transfers/"+createResponse.TransferID+"/", + createResponse.ManageToken, + "", + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, getResponse.Code) + require.Contains(t, getResponse.Body.String(), createResponse.TransferID) + + patchBody := `{"ciphertextBase64":"` + base64.StdEncoding.EncodeToString([]byte("updated-manifest")) + `"}` + patchResponse := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPatch, + "/api/v1/transfers/"+createResponse.TransferID+"/", + createResponse.ManageToken, + patchBody, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, patchResponse.Code) + + updatedTransfer, err := repository.GetTransfer(context.Background(), createResponse.TransferID) + require.NoError(t, err) + require.Equal(t, "transfers/"+createResponse.TransferID+"/manifest.bin", updatedTransfer.ManifestObjectKey) + require.Equal(t, int64(len("updated-manifest")), updatedTransfer.ManifestCiphertextSize) + + resumeResponse := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodGet, + "/api/v1/transfers/"+createResponse.TransferID+"/resume", + createResponse.ManageToken, + "", + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, resumeResponse.Code) + require.Contains(t, resumeResponse.Body.String(), createResponse.TransferID) + + deleteResponse := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodDelete, + "/api/v1/transfers/"+createResponse.TransferID+"/", + createResponse.ManageToken, + "", + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusNoContent, deleteResponse.Code) + + deletedTransfer, err := repository.GetTransfer(context.Background(), createResponse.TransferID) + require.NoError(t, err) + require.Equal(t, models.TransferStatusDeleted, deletedTransfer.Status) + require.NotNil(t, deletedTransfer.DeletedAt) +} + +func TestTransferLifecycleEndpointsWithInMemoryDependencies(t *testing.T) { + t.Parallel() + + router := newTestRouter() + + createResponse := struct { + TransferID string `json:"transferId"` + ManageToken string `json:"manageToken"` + }{} + create := performJSONRequest(t, router, nethttp.MethodPost, "/api/v1/transfers/", `{"expiresInSeconds":3600}`, "198.51.100.42:40001") + require.Equal(t, nethttp.StatusCreated, create.Code) + require.NoError(t, json.Unmarshal(create.Body.Bytes(), &createResponse)) + + register := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/transfers/"+createResponse.TransferID+"/files", + createResponse.ManageToken, + `[{"fileId":"file-a","totalChunks":2,"ciphertextBytes":64,"plaintextBytes":32,"chunkSize":32}]`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, register.Code) + + uploadURLs := struct { + Items []struct { + FileID string `json:"fileId"` + ChunkIndex int `json:"chunkIndex"` + ObjectKey string `json:"objectKey"` + URL string `json:"url"` + } `json:"items"` + }{} + createUploads := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/transfers/"+createResponse.TransferID+"/upload-urls", + createResponse.ManageToken, + `{"chunks":[{"fileId":"file-a","chunkIndex":0},{"fileId":"file-a","chunkIndex":1}]}`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, createUploads.Code) + require.NoError(t, json.Unmarshal(createUploads.Body.Bytes(), &uploadURLs)) + require.Len(t, uploadURLs.Items, 2) + require.Contains(t, uploadURLs.Items[0].URL, uploadURLs.Items[0].ObjectKey) + + complete := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/transfers/"+createResponse.TransferID+"/chunks/complete", + createResponse.ManageToken, + `[{"fileId":"file-a","chunkIndex":0,"ciphertextSize":32,"checksumSha256":"a"},{"fileId":"file-a","chunkIndex":1,"ciphertextSize":32,"checksumSha256":"b"}]`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, complete.Code) + + putManifest := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/transfers/"+createResponse.TransferID+"/manifest", + createResponse.ManageToken, + `{"ciphertextBase64":"`+base64.StdEncoding.EncodeToString([]byte("manifest"))+`"}`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, putManifest.Code) + + finalize := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/transfers/"+createResponse.TransferID+"/finalize", + createResponse.ManageToken, + `{"wrappedRootKey":"wrapped","totalFiles":1,"totalCiphertextBytes":64}`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, finalize.Code) + + publicTransfer := struct { + Status string `json:"status"` + ManifestURL string `json:"manifestUrl"` + WrappedRootKey string `json:"wrappedRootKey"` + }{} + getPublic := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodGet, + "/api/v1/public/transfers/"+createResponse.TransferID+"/", + "", + "", + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, getPublic.Code) + require.NoError(t, json.Unmarshal(getPublic.Body.Bytes(), &publicTransfer)) + require.Equal(t, "ready", publicTransfer.Status) + require.Contains(t, publicTransfer.ManifestURL, "transfers/"+createResponse.TransferID+"/manifest.bin") + require.Equal(t, "wrapped", publicTransfer.WrappedRootKey) + + downloadURLs := struct { + Items []struct { + FileID string `json:"fileId"` + ChunkIndex int `json:"chunkIndex"` + URL string `json:"url"` + } `json:"items"` + }{} + createDownloads := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/public/transfers/"+createResponse.TransferID+"/download-urls", + "", + `{"chunks":[{"fileId":"file-a","chunkIndex":1}]}`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusOK, createDownloads.Code) + require.NoError(t, json.Unmarshal(createDownloads.Body.Bytes(), &downloadURLs)) + require.Len(t, downloadURLs.Items, 1) + require.Contains(t, downloadURLs.Items[0].URL, "transfers/"+createResponse.TransferID+"/files/file-a/chunks/00000001.bin") +} + +func TestManageTransferEndpointsRejectInvalidPayloadsAndMissingTokens(t *testing.T) { + t.Parallel() + + router := newTestRouter() + + createResponse := struct { + TransferID string `json:"transferId"` + ManageToken string `json:"manageToken"` + }{} + create := performJSONRequest(t, router, nethttp.MethodPost, "/api/v1/transfers/", `{"expiresInSeconds":3600}`, "198.51.100.42:40001") + require.Equal(t, nethttp.StatusCreated, create.Code) + require.NoError(t, json.Unmarshal(create.Body.Bytes(), &createResponse)) + + patchResponse := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPatch, + "/api/v1/transfers/"+createResponse.TransferID+"/", + createResponse.ManageToken, + `{"expiresInSeconds":3600}{"extra":true}`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusBadRequest, patchResponse.Code) + require.Contains(t, patchResponse.Body.String(), `"error":"invalid_json"`) + + getResponse := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodGet, + "/api/v1/transfers/"+createResponse.TransferID+"/", + "", + "", + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusUnauthorized, getResponse.Code) + require.Contains(t, getResponse.Body.String(), `"error":"missing_manage_token"`) + + patchUnauthorized := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPatch, + "/api/v1/transfers/"+createResponse.TransferID+"/", + "", + `{"expiresInSeconds":3600}`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusUnauthorized, patchUnauthorized.Code) + require.Contains(t, patchUnauthorized.Body.String(), `"error":"missing_manage_token"`) +} + +func TestTransferEndpointsRejectInvalidJSON(t *testing.T) { + t.Parallel() + + router := newTestRouter() + + createResponse := struct { + TransferID string `json:"transferId"` + ManageToken string `json:"manageToken"` + }{} + create := performJSONRequest(t, router, nethttp.MethodPost, "/api/v1/transfers/", `{"expiresInSeconds":3600}`, "198.51.100.42:40001") + require.Equal(t, nethttp.StatusCreated, create.Code) + require.NoError(t, json.Unmarshal(create.Body.Bytes(), &createResponse)) + + testCases := []struct { + name string + method string + path string + body string + }{ + {name: "register files", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/files", body: `{"broken":true}`}, + {name: "create upload urls", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/upload-urls", body: `{"chunks":"broken"}`}, + {name: "complete chunks", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/chunks/complete", body: `{"broken":true}`}, + {name: "put manifest", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/manifest", body: `{"ciphertextBase64":123}`}, + {name: "finalize", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/finalize", body: `{"wrappedRootKey":123}`}, + {name: "download urls", method: nethttp.MethodPost, path: "/api/v1/public/transfers/" + createResponse.TransferID + "/download-urls", body: `{"chunks":"broken"}`}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + bearer := createResponse.ManageToken + if tc.name == "download urls" { + bearer = "" + } + response := performAuthorizedJSONRequest(t, router, tc.method, tc.path, bearer, tc.body, "198.51.100.42:40001") + require.Equal(t, nethttp.StatusBadRequest, response.Code) + require.Contains(t, response.Body.String(), `"error":"invalid_json"`) + }) + } +} + +func TestTransferEndpointsSurfaceServiceErrors(t *testing.T) { + t.Parallel() + + router := newTestRouter() + + createResponse := struct { + TransferID string `json:"transferId"` + ManageToken string `json:"manageToken"` + }{} + create := performJSONRequest(t, router, nethttp.MethodPost, "/api/v1/transfers/", `{"expiresInSeconds":3600}`, "198.51.100.42:40001") + require.Equal(t, nethttp.StatusCreated, create.Code) + require.NoError(t, json.Unmarshal(create.Body.Bytes(), &createResponse)) + + managePaths := []struct { + name string + method string + path string + body string + }{ + {name: "register files", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/files", body: `[{"fileId":"file-a","totalChunks":1,"ciphertextBytes":32,"chunkSize":32}]`}, + {name: "create upload urls", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/upload-urls", body: `{"chunks":[{"fileId":"file-a","chunkIndex":0}]}`}, + {name: "complete chunks", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/chunks/complete", body: `[{"fileId":"file-a","chunkIndex":0,"ciphertextSize":32,"checksumSha256":"a"}]`}, + {name: "put manifest", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/manifest", body: `{"ciphertextBase64":"` + base64.StdEncoding.EncodeToString([]byte("manifest")) + `"}`}, + {name: "finalize", method: nethttp.MethodPost, path: "/api/v1/transfers/" + createResponse.TransferID + "/finalize", body: `{"wrappedRootKey":"wrapped","totalFiles":1,"totalCiphertextBytes":64}`}, + {name: "delete transfer", method: nethttp.MethodDelete, path: "/api/v1/transfers/" + createResponse.TransferID + "/", body: ``}, + {name: "resume transfer", method: nethttp.MethodGet, path: "/api/v1/transfers/" + createResponse.TransferID + "/resume", body: ``}, + } + + for _, tc := range managePaths { + tc := tc + t.Run(tc.name, func(t *testing.T) { + response := performAuthorizedJSONRequest(t, router, tc.method, tc.path, "", tc.body, "198.51.100.42:40001") + require.Equal(t, nethttp.StatusUnauthorized, response.Code) + require.Contains(t, response.Body.String(), `"error":"missing_manage_token"`) + }) + } + + getPublic := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodGet, + "/api/v1/public/transfers/missing-transfer/", + "", + "", + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusNotFound, getPublic.Code) + require.Contains(t, getPublic.Body.String(), `"error":"not_found"`) + + createDownloads := performAuthorizedJSONRequest( + t, + router, + nethttp.MethodPost, + "/api/v1/public/transfers/missing-transfer/download-urls", + "", + `{"chunks":[{"fileId":"file-a","chunkIndex":0}]}`, + "198.51.100.42:40001", + ) + require.Equal(t, nethttp.StatusNotFound, createDownloads.Code) + require.Contains(t, createDownloads.Body.String(), `"error":"not_found"`) +} + +func TestWriteErrorAndHeaderHelpers(t *testing.T) { + t.Parallel() + + t.Run("write error handles generic failures", func(t *testing.T) { + t.Parallel() + + response := httptest.NewRecorder() + writeError(response, errors.New("boom")) + + require.Equal(t, nethttp.StatusInternalServerError, response.Code) + require.Contains(t, response.Body.String(), `"error":"internal_error"`) + }) + + t.Run("bearer token trims and validates prefixes", func(t *testing.T) { + t.Parallel() + + request := httptest.NewRequest(nethttp.MethodGet, "/healthz", nil) + request.Header.Set("Authorization", " Bearer test-token ") + require.Equal(t, "test-token", bearerToken(request)) + + request.Header.Set("Authorization", "Token test-token") + require.Empty(t, bearerToken(request)) + }) + + t.Run("client key handles empty, hostport and raw values", func(t *testing.T) { + t.Parallel() + + request := httptest.NewRequest(nethttp.MethodGet, "/healthz", nil) + request.RemoteAddr = "198.51.100.42:41000" + require.Equal(t, "198.51.100.42", clientKey(request)) + + request.RemoteAddr = "198.51.100.42" + require.Equal(t, "198.51.100.42", clientKey(request)) + + request.RemoteAddr = "" + require.Empty(t, clientKey(request)) + }) +} + +func newTestRouterWithRepository() (nethttp.Handler, *routerRepository) { + cfg := testRouterConfig() + repository := newRouterRepository() + router := NewRouter( + cfg, + slog.New(slog.NewTextHandler(io.Discard, nil)), + service.New(cfg, repository, &routerStorage{}, ratelimit.NewMemoryLimiter()), + ) + return router, repository +} + +func performAuthorizedJSONRequest(t *testing.T, handler nethttp.Handler, method string, path string, bearer string, body string, remoteAddr string) *httptest.ResponseRecorder { + t.Helper() + + request := httptest.NewRequest(method, path, bytes.NewBufferString(body)) + if body != "" { + request.Header.Set("Content-Type", "application/json") + } + if bearer != "" { + request.Header.Set("Authorization", "Bearer "+bearer) + } + request.RemoteAddr = remoteAddr + + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + return response +} diff --git a/apps/api/internal/jobs/expiry.go b/apps/api/internal/jobs/expiry.go new file mode 100644 index 0000000..30af305 --- /dev/null +++ b/apps/api/internal/jobs/expiry.go @@ -0,0 +1,26 @@ +package jobs + +import ( + "context" + "log/slog" + "time" + + "github.com/xdrop/monorepo/internal/service" +) + +// StartCleanup periodically purges expired or deleted transfers until the context is canceled. +func StartCleanup(ctx context.Context, logger *slog.Logger, interval time.Duration, svc *service.Service) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := svc.CleanupExpired(ctx); err != nil { + logger.Error("cleanup tick failed", "error", err) + } + } + } +} diff --git a/apps/api/internal/jobs/expiry_test.go b/apps/api/internal/jobs/expiry_test.go new file mode 100644 index 0000000..0a23fa7 --- /dev/null +++ b/apps/api/internal/jobs/expiry_test.go @@ -0,0 +1,223 @@ +package jobs + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/config" + "github.com/xdrop/monorepo/internal/models" + "github.com/xdrop/monorepo/internal/repo" + "github.com/xdrop/monorepo/internal/service" +) + +func TestStartCleanupPurgesExpiredTransfersOnTick(t *testing.T) { + t.Parallel() + + cleanupRepo := &cleanupRepository{ + transfer: models.Transfer{ + ID: "expired-transfer", + Status: models.TransferStatusReady, + ExpiresAt: time.Now().UTC().Add(-time.Hour), + }, + purged: make(chan struct{}), + } + objectStorage := &cleanupStorage{deletedPrefixes: make(chan string, 1)} + svc := service.New(config.Config{}, cleanupRepo, objectStorage, nil) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan struct{}) + go func() { + StartCleanup(ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), 10*time.Millisecond, svc) + close(done) + }() + + select { + case prefix := <-objectStorage.deletedPrefixes: + require.Equal(t, "transfers/expired-transfer/", prefix) + case <-time.After(time.Second): + t.Fatal("cleanup job did not delete expired transfer objects") + } + + select { + case <-cleanupRepo.purged: + case <-time.After(time.Second): + t.Fatal("cleanup job did not mark transfer as purged") + } + + cancel() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("cleanup job did not stop after cancellation") + } +} + +func TestStartCleanupStopsWhenContextIsCancelled(t *testing.T) { + t.Parallel() + + svc := service.New(config.Config{}, &cleanupRepository{purged: make(chan struct{})}, &cleanupStorage{ + deletedPrefixes: make(chan string, 1), + }, nil) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + StartCleanup(ctx, slog.New(slog.NewTextHandler(io.Discard, nil)), time.Hour, svc) + close(done) + }() + + cancel() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("cleanup job did not exit after cancellation") + } +} + +func TestStartCleanupLogsCleanupErrors(t *testing.T) { + t.Parallel() + + var buffer bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buffer, nil)) + svc := service.New(config.Config{}, &cleanupRepository{ + listErr: errors.New("cleanup failed"), + purged: make(chan struct{}), + }, &cleanupStorage{ + deletedPrefixes: make(chan string, 1), + }, nil) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + StartCleanup(ctx, logger, 10*time.Millisecond, svc) + close(done) + }() + + require.Eventually(t, func() bool { + return strings.Contains(buffer.String(), "cleanup tick failed") + }, time.Second, 10*time.Millisecond) + + cancel() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("cleanup job did not stop after logging an error") + } +} + +type cleanupRepository struct { + mu sync.Mutex + listErr error + transfer models.Transfer + purged chan struct{} + once sync.Once +} + +func (r *cleanupRepository) CreateTransfer(context.Context, models.Transfer) error { + return nil +} + +func (r *cleanupRepository) GetTransfer(context.Context, string) (models.Transfer, error) { + return models.Transfer{}, repo.ErrNotFound +} + +func (r *cleanupRepository) RegisterFiles(context.Context, string, []models.TransferFile) error { + return nil +} + +func (r *cleanupRepository) ListFiles(context.Context, string) ([]models.TransferFile, error) { + return nil, nil +} + +func (r *cleanupRepository) CompleteChunks(context.Context, string, []models.TransferChunk) error { + return nil +} + +func (r *cleanupRepository) GetResumeState(context.Context, string) (models.TransferResumeState, error) { + return models.TransferResumeState{}, nil +} + +func (r *cleanupRepository) SetManifest(context.Context, string, string, int64) error { + return nil +} + +func (r *cleanupRepository) FinalizeTransfer(context.Context, string, string, int, int64) error { + return nil +} + +func (r *cleanupRepository) UpdateTransfer(context.Context, string, models.UpdateTransferParams) error { + return nil +} + +func (r *cleanupRepository) MarkDeleted(context.Context, string) error { + return nil +} + +func (r *cleanupRepository) ListCleanupCandidates(context.Context, int) ([]models.Transfer, error) { + if r.listErr != nil { + return nil, r.listErr + } + + r.mu.Lock() + defer r.mu.Unlock() + + if r.transfer.ID == "" || r.transfer.PurgedAt != nil { + return nil, nil + } + + return []models.Transfer{r.transfer}, nil +} + +func (r *cleanupRepository) MarkPurged(context.Context, string) error { + r.mu.Lock() + defer r.mu.Unlock() + + now := time.Now().UTC() + r.transfer.PurgedAt = &now + r.transfer.Status = models.TransferStatusExpired + r.once.Do(func() { + close(r.purged) + }) + return nil +} + +type cleanupStorage struct { + deletedPrefixes chan string +} + +func (s *cleanupStorage) PresignUpload(context.Context, string, time.Duration) (string, error) { + return "", nil +} + +func (s *cleanupStorage) PresignDownload(context.Context, string, time.Duration) (string, error) { + return "", nil +} + +func (s *cleanupStorage) PutObject(context.Context, string, []byte, string) error { + return nil +} + +func (s *cleanupStorage) DeletePrefix(_ context.Context, prefix string) error { + select { + case s.deletedPrefixes <- prefix: + default: + } + return nil +} + +func (s *cleanupStorage) EnsureBucket(context.Context) error { + return nil +} diff --git a/apps/api/internal/models/models.go b/apps/api/internal/models/models.go new file mode 100644 index 0000000..0e87081 --- /dev/null +++ b/apps/api/internal/models/models.go @@ -0,0 +1,80 @@ +package models + +import "time" + +// TransferStatus describes the lifecycle stage of a transfer in persistent storage. +type TransferStatus string + +const ( + // Draft transfers exist but do not have any registered files yet. + TransferStatusDraft TransferStatus = "draft" + // Uploading transfers have registered files and may still be receiving chunks. + TransferStatusUploading TransferStatus = "uploading" + // Ready transfers have a manifest, wrapped root key, and all chunks uploaded. + TransferStatusReady TransferStatus = "ready" + // Incomplete transfers are visible to the public API but not yet downloadable. + TransferStatusIncomplete TransferStatus = "incomplete" + // Expired transfers have passed their retention window and await cleanup. + TransferStatusExpired TransferStatus = "expired" + // Deleted transfers were explicitly removed by their creator. + TransferStatusDeleted TransferStatus = "deleted" + // Failed transfers represent interrupted local state that never became ready. + TransferStatusFailed TransferStatus = "failed" +) + +// Transfer stores transfer-level metadata and lifecycle state. +type Transfer struct { + ID string + Status TransferStatus + WrappedRootKey string + ManifestObjectKey string + ManifestCiphertextSize int64 + TotalFiles int + TotalCiphertextBytes int64 + ExpiresAt time.Time + CreatedAt time.Time + UpdatedAt time.Time + FinalizedAt *time.Time + ManageTokenHash string + DeletedAt *time.Time + PurgedAt *time.Time +} + +// TransferFile stores file-level metadata for a transfer. +type TransferFile struct { + TransferID string + OpaqueFileID string + TotalChunks int + CiphertextBytes int64 + PlaintextBytes *int64 + ChunkSize int64 + UploadStatus string + CreatedAt time.Time + UpdatedAt time.Time +} + +// TransferChunk tracks each uploaded ciphertext chunk. +type TransferChunk struct { + TransferID string + OpaqueFileID string + ChunkIndex int + ObjectKey string + CiphertextSize int64 + ChecksumSHA256 string + UploadedAt time.Time +} + +// TransferResumeState combines a transfer with file metadata and completed chunk indexes. +type TransferResumeState struct { + Transfer Transfer + Files []TransferFile + UploadedChunks map[string][]int +} + +// UpdateTransferParams describes the mutable fields the service can update after creation. +type UpdateTransferParams struct { + ManifestObjectKey *string + ExpiresAt *time.Time + EncryptedManifest []byte + ManifestCiphertextSize *int64 +} diff --git a/apps/api/internal/ratelimit/limiter.go b/apps/api/internal/ratelimit/limiter.go new file mode 100644 index 0000000..be973cd --- /dev/null +++ b/apps/api/internal/ratelimit/limiter.go @@ -0,0 +1,82 @@ +package ratelimit + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/redis/go-redis/v9" +) + +// Limiter decides whether a caller may proceed within a given fixed time window. +type Limiter interface { + Allow(ctx context.Context, key string, limit int, window time.Duration) (bool, error) +} + +// RedisLimiter stores counters in Redis so limits can be shared across API instances. +type RedisLimiter struct { + client *redis.Client +} + +// NewRedisLimiter builds a Redis-backed limiter for production deployments. +func NewRedisLimiter(client *redis.Client) *RedisLimiter { + return &RedisLimiter{client: client} +} + +// Allow increments the caller key and compares the result with the configured limit. +func (l *RedisLimiter) Allow(ctx context.Context, key string, limit int, window time.Duration) (bool, error) { + if limit <= 0 { + return true, nil + } + + pipeline := l.client.TxPipeline() + countCmd := pipeline.Incr(ctx, key) + pipeline.Expire(ctx, key, window) + + if _, err := pipeline.Exec(ctx); err != nil { + return false, fmt.Errorf("exec rate limit pipeline: %w", err) + } + + return countCmd.Val() <= int64(limit), nil +} + +// MemoryLimiter provides an in-process limiter for tests and single-node development. +type MemoryLimiter struct { + mu sync.Mutex + entries map[string]memoryEntry +} + +type memoryEntry struct { + count int + deadline time.Time +} + +// NewMemoryLimiter builds an empty in-memory limiter state. +func NewMemoryLimiter() *MemoryLimiter { + return &MemoryLimiter{ + entries: map[string]memoryEntry{}, + } +} + +// Allow applies the same fixed-window semantics as RedisLimiter without external state. +func (l *MemoryLimiter) Allow(_ context.Context, key string, limit int, window time.Duration) (bool, error) { + if limit <= 0 { + return true, nil + } + + now := time.Now() + + l.mu.Lock() + defer l.mu.Unlock() + + entry, ok := l.entries[key] + if !ok || now.After(entry.deadline) { + entry = memoryEntry{count: 0, deadline: now.Add(window)} + } + + entry.count++ + l.entries[key] = entry + + return entry.count <= limit, nil +} diff --git a/apps/api/internal/ratelimit/limiter_integration_test.go b/apps/api/internal/ratelimit/limiter_integration_test.go new file mode 100644 index 0000000..1198e37 --- /dev/null +++ b/apps/api/internal/ratelimit/limiter_integration_test.go @@ -0,0 +1,82 @@ +package ratelimit + +import ( + "context" + "fmt" + "os/exec" + "testing" + "time" + + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func TestRedisLimiterBlocksThenResetsAfterWindow(t *testing.T) { + skipIfDockerUnavailable(t) + + ctx := context.Background() + client := startRedisClient(t, ctx) + limiter := NewRedisLimiter(client) + + key := "rate-limit:test" + allowed, err := limiter.Allow(ctx, key, 1, 2*time.Second) + require.NoError(t, err) + require.True(t, allowed) + + allowed, err = limiter.Allow(ctx, key, 1, 2*time.Second) + require.NoError(t, err) + require.False(t, allowed) + + time.Sleep(2200 * time.Millisecond) + + allowed, err = limiter.Allow(ctx, key, 1, 2*time.Second) + require.NoError(t, err) + require.True(t, allowed) +} + +func startRedisClient(t *testing.T, ctx context.Context) *redis.Client { + t.Helper() + + container, err := testcontainers.Run( + ctx, + "redis:7-alpine", + testcontainers.WithExposedPorts("6379/tcp"), + testcontainers.WithWaitStrategy( + wait.ForLog("Ready to accept connections").WithStartupTimeout(60*time.Second), + ), + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, testcontainers.TerminateContainer(container)) + }) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "6379/tcp") + require.NoError(t, err) + + client := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%s", host, port.Port()), + DB: 0, + }) + require.NoError(t, client.Ping(ctx).Err()) + t.Cleanup(func() { + require.NoError(t, client.Close()) + }) + + return client +} + +func skipIfDockerUnavailable(t *testing.T) { + t.Helper() + + if testing.Short() { + t.Skip("skipping docker-backed integration test in short mode") + } + + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skipf("skipping docker-backed integration test: %v", err) + } +} diff --git a/apps/api/internal/ratelimit/limiter_test.go b/apps/api/internal/ratelimit/limiter_test.go new file mode 100644 index 0000000..37f8524 --- /dev/null +++ b/apps/api/internal/ratelimit/limiter_test.go @@ -0,0 +1,86 @@ +package ratelimit + +import ( + "context" + "testing" + "time" + + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func TestMemoryLimiterAllowsUpToLimitThenBlocks(t *testing.T) { + t.Parallel() + + limiter := NewMemoryLimiter() + + allowed, err := limiter.Allow(context.Background(), "client-a", 2, time.Minute) + require.NoError(t, err) + require.True(t, allowed) + + allowed, err = limiter.Allow(context.Background(), "client-a", 2, time.Minute) + require.NoError(t, err) + require.True(t, allowed) + + allowed, err = limiter.Allow(context.Background(), "client-a", 2, time.Minute) + require.NoError(t, err) + require.False(t, allowed) +} + +func TestMemoryLimiterResetsAfterWindow(t *testing.T) { + t.Parallel() + + limiter := NewMemoryLimiter() + + allowed, err := limiter.Allow(context.Background(), "client-a", 1, 10*time.Millisecond) + require.NoError(t, err) + require.True(t, allowed) + + allowed, err = limiter.Allow(context.Background(), "client-a", 1, 10*time.Millisecond) + require.NoError(t, err) + require.False(t, allowed) + + time.Sleep(25 * time.Millisecond) + + allowed, err = limiter.Allow(context.Background(), "client-a", 1, 10*time.Millisecond) + require.NoError(t, err) + require.True(t, allowed) +} + +func TestMemoryLimiterAllowsNonPositiveLimits(t *testing.T) { + t.Parallel() + + limiter := NewMemoryLimiter() + + for _, limit := range []int{0, -1} { + allowed, err := limiter.Allow(context.Background(), "client-a", limit, time.Minute) + require.NoError(t, err) + require.True(t, allowed) + } +} + +func TestRedisLimiterAllowsNonPositiveLimits(t *testing.T) { + t.Parallel() + + allowed, err := NewRedisLimiter(nil).Allow(context.Background(), "client-a", 0, time.Minute) + require.NoError(t, err) + require.True(t, allowed) +} + +func TestRedisLimiterReturnsPipelineErrors(t *testing.T) { + t.Parallel() + + client := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:1", + DialTimeout: 20 * time.Millisecond, + ReadTimeout: 20 * time.Millisecond, + WriteTimeout: 20 * time.Millisecond, + }) + t.Cleanup(func() { + _ = client.Close() + }) + + allowed, err := NewRedisLimiter(client).Allow(context.Background(), "client-a", 1, time.Minute) + require.False(t, allowed) + require.ErrorContains(t, err, "exec rate limit pipeline") +} diff --git a/apps/api/internal/repo/migrate.go b/apps/api/internal/repo/migrate.go new file mode 100644 index 0000000..547e714 --- /dev/null +++ b/apps/api/internal/repo/migrate.go @@ -0,0 +1,93 @@ +package repo + +import ( + "context" + "embed" + "fmt" + "io/fs" + "path/filepath" + "sort" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +var ( + embeddedMigrationFiles migrationFiles = migrationFS + newMigrationDBFromPool = func(db *pgxpool.Pool) postgresDB { + return pgxPoolDB{pool: db} + } +) + +// RunMigrations applies embedded SQL migrations in lexical order and records each one once. +func RunMigrations(ctx context.Context, db *pgxpool.Pool) error { + return runMigrations(ctx, newMigrationDBFromPool(db), embeddedMigrationFiles) +} + +// migrationFiles abstracts embedded files so migration tests can inject custom fixtures. +type migrationFiles interface { + ReadDir(name string) ([]fs.DirEntry, error) + ReadFile(name string) ([]byte, error) +} + +// runMigrations ensures each migration is executed inside its own transaction. +func runMigrations(ctx context.Context, db postgresDB, files migrationFiles) error { + if err := db.Exec(ctx, ` + CREATE TABLE IF NOT EXISTS schema_migrations ( + name text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + `); err != nil { + return fmt.Errorf("create schema_migrations: %w", err) + } + + entries, err := files.ReadDir("migrations") + if err != nil { + return fmt.Errorf("read migrations: %w", err) + } + + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + names = append(names, entry.Name()) + } + sort.Strings(names) + + for _, name := range names { + var exists bool + if err := db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE name = $1)`, name).Scan(&exists); err != nil { + return fmt.Errorf("query migration %s: %w", name, err) + } + if exists { + continue + } + + payload, err := files.ReadFile(filepath.ToSlash(filepath.Join("migrations", name))) + if err != nil { + return fmt.Errorf("read migration %s: %w", name, err) + } + + tx, err := db.Begin(ctx) + if err != nil { + return fmt.Errorf("begin migration %s: %w", name, err) + } + + if err = tx.Exec(ctx, string(payload)); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("apply migration %s: %w", name, err) + } + if err = tx.Exec(ctx, `INSERT INTO schema_migrations (name) VALUES ($1)`, name); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("record migration %s: %w", name, err) + } + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("commit migration %s: %w", name, err) + } + } + + return nil +} diff --git a/apps/api/internal/repo/migrate_unit_test.go b/apps/api/internal/repo/migrate_unit_test.go new file mode 100644 index 0000000..ea3b890 --- /dev/null +++ b/apps/api/internal/repo/migrate_unit_test.go @@ -0,0 +1,222 @@ +package repo + +import ( + "context" + "errors" + "io/fs" + "strings" + "testing" + "testing/fstest" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" +) + +func TestRunMigrationsUnit(t *testing.T) { + t.Parallel() + + t.Run("create schema failure", func(t *testing.T) { + err := runMigrations(context.Background(), stubPostgresDB{ + execFn: func(context.Context, string, ...any) error { + return errors.New("boom") + }, + }, fstest.MapFS{}) + require.ErrorContains(t, err, "create schema_migrations") + }) + + t.Run("read dir failure", func(t *testing.T) { + err := runMigrations(context.Background(), stubPostgresDB{}, stubMigrationFiles{ + readDirFn: func(string) ([]fs.DirEntry, error) { + return nil, errors.New("boom") + }, + }) + require.ErrorContains(t, err, "read migrations") + }) + + t.Run("query existing migration failure", func(t *testing.T) { + err := runMigrations(context.Background(), stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{err: errors.New("boom")} + }, + }, fstest.MapFS{ + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + }) + require.ErrorContains(t, err, "query migration 001_first.sql") + }) + + t.Run("read migration failure", func(t *testing.T) { + err := runMigrations(context.Background(), stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{false}} + }, + }, stubMigrationFiles{ + readDirFn: func(string) ([]fs.DirEntry, error) { + return fstest.MapFS{ + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + }.ReadDir("migrations") + }, + readFileFn: func(string) ([]byte, error) { + return nil, errors.New("boom") + }, + }) + require.ErrorContains(t, err, "read migration 001_first.sql") + }) + + t.Run("begin migration failure", func(t *testing.T) { + err := runMigrations(context.Background(), stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{false}} + }, + beginFn: func(context.Context) (postgresTx, error) { + return nil, errors.New("boom") + }, + }, fstest.MapFS{ + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + }) + require.ErrorContains(t, err, "begin migration 001_first.sql") + }) + + t.Run("apply migration failure", func(t *testing.T) { + tx := &stubPostgresTx{ + execFn: func(context.Context, string, ...any) error { + return errors.New("boom") + }, + } + + err := runMigrations(context.Background(), stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{false}} + }, + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }, fstest.MapFS{ + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + }) + + require.ErrorContains(t, err, "apply migration 001_first.sql") + require.True(t, tx.rollbackCalled) + }) + + t.Run("record migration failure", func(t *testing.T) { + tx := &stubPostgresTx{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "INSERT INTO schema_migrations") { + return errors.New("boom") + } + return nil + }, + } + + err := runMigrations(context.Background(), stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{false}} + }, + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }, fstest.MapFS{ + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + }) + + require.ErrorContains(t, err, "record migration 001_first.sql") + require.True(t, tx.rollbackCalled) + }) + + t.Run("commit migration failure", func(t *testing.T) { + tx := &stubPostgresTx{commitErr: errors.New("boom")} + + err := runMigrations(context.Background(), stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{false}} + }, + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }, fstest.MapFS{ + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + }) + + require.ErrorContains(t, err, "commit migration 001_first.sql") + }) + + t.Run("success sorts and skips existing migrations", func(t *testing.T) { + applied := []string{} + checked := []string{} + + tx := &stubPostgresTx{ + execFn: func(_ context.Context, sql string, args ...any) error { + if strings.Contains(sql, "INSERT INTO schema_migrations") { + applied = append(applied, args[0].(string)) + } + return nil + }, + } + + err := runMigrations(context.Background(), stubPostgresDB{ + queryRowFn: func(_ context.Context, _ string, args ...any) postgresRow { + name := args[0].(string) + checked = append(checked, name) + return stubRow{values: []any{name == "001_first.sql"}} + }, + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }, fstest.MapFS{ + "migrations/002_second.sql": &fstest.MapFile{Data: []byte("select 2;")}, + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + }) + + require.NoError(t, err) + require.Equal(t, []string{"001_first.sql", "002_second.sql"}, checked) + require.Equal(t, []string{"002_second.sql"}, applied) + }) + + t.Run("ignores directory entries", func(t *testing.T) { + tx := &stubPostgresTx{} + + err := runMigrations(context.Background(), stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{false}} + }, + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }, fstest.MapFS{ + "migrations/nested": &fstest.MapFile{Mode: fs.ModeDir}, + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + "migrations/nested/skip.sql": &fstest.MapFile{Data: []byte("select 2;")}, + }) + + require.NoError(t, err) + require.True(t, tx.commitCalled) + }) + + t.Run("RunMigrations uses the pool wrapper and embedded files", func(t *testing.T) { + originalFiles := embeddedMigrationFiles + originalFactory := newMigrationDBFromPool + t.Cleanup(func() { + embeddedMigrationFiles = originalFiles + newMigrationDBFromPool = originalFactory + }) + + embeddedMigrationFiles = fstest.MapFS{ + "migrations/001_first.sql": &fstest.MapFile{Data: []byte("select 1;")}, + } + + tx := &stubPostgresTx{} + newMigrationDBFromPool = func(*pgxpool.Pool) postgresDB { + return stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{false}} + }, + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + } + } + + require.NoError(t, RunMigrations(context.Background(), nil)) + require.True(t, tx.commitCalled) + }) +} diff --git a/apps/api/internal/repo/migrations/001_init.sql b/apps/api/internal/repo/migrations/001_init.sql new file mode 100644 index 0000000..91f4b18 --- /dev/null +++ b/apps/api/internal/repo/migrations/001_init.sql @@ -0,0 +1,46 @@ +CREATE TABLE IF NOT EXISTS transfers ( + id text PRIMARY KEY, + status text NOT NULL, + wrapped_root_key text NOT NULL DEFAULT '', + manifest_object_key text NOT NULL DEFAULT '', + manifest_ciphertext_size bigint NOT NULL DEFAULT 0, + total_files integer NOT NULL DEFAULT 0, + total_ciphertext_bytes bigint NOT NULL DEFAULT 0, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + finalized_at timestamptz, + manage_token_hash text NOT NULL, + deleted_at timestamptz, + purged_at timestamptz +); + +CREATE TABLE IF NOT EXISTS transfer_files ( + id bigserial PRIMARY KEY, + transfer_id text NOT NULL REFERENCES transfers(id) ON DELETE CASCADE, + opaque_file_id text NOT NULL, + total_chunks integer NOT NULL, + ciphertext_bytes bigint NOT NULL, + plaintext_bytes bigint, + chunk_size bigint NOT NULL, + upload_status text NOT NULL DEFAULT 'pending', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (transfer_id, opaque_file_id) +); + +CREATE TABLE IF NOT EXISTS transfer_chunks ( + id bigserial PRIMARY KEY, + transfer_id text NOT NULL REFERENCES transfers(id) ON DELETE CASCADE, + opaque_file_id text NOT NULL, + chunk_index integer NOT NULL, + object_key text NOT NULL, + ciphertext_size bigint NOT NULL, + checksum_sha256 text NOT NULL, + uploaded_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (transfer_id, opaque_file_id, chunk_index) +); + +CREATE INDEX IF NOT EXISTS idx_transfers_status_expires ON transfers(status, expires_at); +CREATE INDEX IF NOT EXISTS idx_transfer_files_transfer_id ON transfer_files(transfer_id); +CREATE INDEX IF NOT EXISTS idx_transfer_chunks_transfer_file ON transfer_chunks(transfer_id, opaque_file_id); diff --git a/apps/api/internal/repo/postgres.go b/apps/api/internal/repo/postgres.go new file mode 100644 index 0000000..6bda89f --- /dev/null +++ b/apps/api/internal/repo/postgres.go @@ -0,0 +1,510 @@ +package repo + +import ( + "context" + "errors" + "fmt" + "slices" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/xdrop/monorepo/internal/models" +) + +// ErrNotFound reports that a requested transfer does not exist. +var ErrNotFound = errors.New("not found") + +type postgresRow interface { + Scan(dest ...any) error +} + +type postgresRows interface { + Close() + Err() error + Next() bool + Scan(dest ...any) error +} + +type postgresTx interface { + Commit(ctx context.Context) error + Exec(ctx context.Context, sql string, args ...any) error + QueryRow(ctx context.Context, sql string, args ...any) postgresRow + Rollback(ctx context.Context) error +} + +type postgresDB interface { + Begin(ctx context.Context) (postgresTx, error) + Exec(ctx context.Context, sql string, args ...any) error + Query(ctx context.Context, sql string, args ...any) (postgresRows, error) + QueryRow(ctx context.Context, sql string, args ...any) postgresRow +} + +type pgxPoolDB struct { + pool *pgxpool.Pool +} + +func (d pgxPoolDB) Begin(ctx context.Context) (postgresTx, error) { + tx, err := d.pool.Begin(ctx) + if err != nil { + return nil, err + } + + return pgxTx{tx: tx}, nil +} + +func (d pgxPoolDB) Exec(ctx context.Context, sql string, args ...any) error { + _, err := d.pool.Exec(ctx, sql, args...) + return err +} + +func (d pgxPoolDB) Query(ctx context.Context, sql string, args ...any) (postgresRows, error) { + return d.pool.Query(ctx, sql, args...) +} + +func (d pgxPoolDB) QueryRow(ctx context.Context, sql string, args ...any) postgresRow { + return d.pool.QueryRow(ctx, sql, args...) +} + +type pgxTx struct { + tx pgx.Tx +} + +func (t pgxTx) Commit(ctx context.Context) error { + return t.tx.Commit(ctx) +} + +func (t pgxTx) Exec(ctx context.Context, sql string, args ...any) error { + _, err := t.tx.Exec(ctx, sql, args...) + return err +} + +func (t pgxTx) QueryRow(ctx context.Context, sql string, args ...any) postgresRow { + return t.tx.QueryRow(ctx, sql, args...) +} + +func (t pgxTx) Rollback(ctx context.Context) error { + return t.tx.Rollback(ctx) +} + +// PostgresRepository stores transfers, files, and chunks in PostgreSQL. +type PostgresRepository struct { + db postgresDB +} + +// NewPostgresRepository wraps a pgx pool with the repository implementation. +func NewPostgresRepository(db *pgxpool.Pool) *PostgresRepository { + return &PostgresRepository{db: pgxPoolDB{pool: db}} +} + +// CreateTransfer inserts a new transfer in draft state. +func (r *PostgresRepository) CreateTransfer(ctx context.Context, transfer models.Transfer) error { + err := r.db.Exec(ctx, ` + INSERT INTO transfers ( + id, status, expires_at, manage_token_hash, created_at, updated_at + ) VALUES ($1, $2, $3, $4, $5, $5) + `, transfer.ID, transfer.Status, transfer.ExpiresAt, transfer.ManageTokenHash, transfer.CreatedAt) + if err != nil { + return fmt.Errorf("insert transfer: %w", err) + } + + return nil +} + +// GetTransfer loads the persisted transfer record for the given identifier. +func (r *PostgresRepository) GetTransfer(ctx context.Context, transferID string) (models.Transfer, error) { + var transfer models.Transfer + + err := r.db.QueryRow(ctx, ` + SELECT + id, status, wrapped_root_key, manifest_object_key, manifest_ciphertext_size, + total_files, total_ciphertext_bytes, expires_at, created_at, updated_at, + finalized_at, manage_token_hash, deleted_at, purged_at + FROM transfers + WHERE id = $1 + `, transferID).Scan( + &transfer.ID, + &transfer.Status, + &transfer.WrappedRootKey, + &transfer.ManifestObjectKey, + &transfer.ManifestCiphertextSize, + &transfer.TotalFiles, + &transfer.TotalCiphertextBytes, + &transfer.ExpiresAt, + &transfer.CreatedAt, + &transfer.UpdatedAt, + &transfer.FinalizedAt, + &transfer.ManageTokenHash, + &transfer.DeletedAt, + &transfer.PurgedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return models.Transfer{}, ErrNotFound + } + if err != nil { + return models.Transfer{}, fmt.Errorf("select transfer: %w", err) + } + + return transfer, nil +} + +// RegisterFiles upserts file metadata and marks the transfer as uploading. +func (r *PostgresRepository) RegisterFiles(ctx context.Context, transferID string, files []models.TransferFile) error { + tx, err := r.db.Begin(ctx) + if err != nil { + return fmt.Errorf("begin register files: %w", err) + } + defer tx.Rollback(ctx) + + for _, file := range files { + err = tx.Exec(ctx, ` + INSERT INTO transfer_files ( + transfer_id, opaque_file_id, total_chunks, ciphertext_bytes, plaintext_bytes, chunk_size, upload_status, created_at, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, now(), now()) + ON CONFLICT (transfer_id, opaque_file_id) DO UPDATE + SET total_chunks = EXCLUDED.total_chunks, + ciphertext_bytes = EXCLUDED.ciphertext_bytes, + plaintext_bytes = EXCLUDED.plaintext_bytes, + chunk_size = EXCLUDED.chunk_size, + updated_at = now() + `, transferID, file.OpaqueFileID, file.TotalChunks, file.CiphertextBytes, file.PlaintextBytes, file.ChunkSize, "pending") + if err != nil { + return fmt.Errorf("insert file %s: %w", file.OpaqueFileID, err) + } + } + + err = tx.Exec(ctx, `UPDATE transfers SET status = $2, updated_at = now() WHERE id = $1`, transferID, models.TransferStatusUploading) + if err != nil { + return fmt.Errorf("set uploading status: %w", err) + } + + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("commit register files: %w", err) + } + + return nil +} + +// ListFiles returns every file registered for a transfer in stable order. +func (r *PostgresRepository) ListFiles(ctx context.Context, transferID string) ([]models.TransferFile, error) { + rows, err := r.db.Query(ctx, ` + SELECT transfer_id, opaque_file_id, total_chunks, ciphertext_bytes, plaintext_bytes, chunk_size, upload_status, created_at, updated_at + FROM transfer_files + WHERE transfer_id = $1 + ORDER BY opaque_file_id + `, transferID) + if err != nil { + return nil, fmt.Errorf("query files: %w", err) + } + defer rows.Close() + + files := []models.TransferFile{} + for rows.Next() { + var file models.TransferFile + if err := rows.Scan( + &file.TransferID, + &file.OpaqueFileID, + &file.TotalChunks, + &file.CiphertextBytes, + &file.PlaintextBytes, + &file.ChunkSize, + &file.UploadStatus, + &file.CreatedAt, + &file.UpdatedAt, + ); err != nil { + return nil, fmt.Errorf("scan file: %w", err) + } + files = append(files, file) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate files: %w", err) + } + + return files, nil +} + +// CompleteChunks records uploaded chunks and advances per-file upload status. +func (r *PostgresRepository) CompleteChunks(ctx context.Context, transferID string, chunks []models.TransferChunk) error { + tx, err := r.db.Begin(ctx) + if err != nil { + return fmt.Errorf("begin complete chunks: %w", err) + } + defer tx.Rollback(ctx) + + filesTouched := []string{} + for _, chunk := range chunks { + err = tx.Exec(ctx, ` + INSERT INTO transfer_chunks ( + transfer_id, opaque_file_id, chunk_index, object_key, ciphertext_size, checksum_sha256, uploaded_at + ) VALUES ($1, $2, $3, $4, $5, $6, now()) + ON CONFLICT (transfer_id, opaque_file_id, chunk_index) DO UPDATE + SET object_key = EXCLUDED.object_key, + ciphertext_size = EXCLUDED.ciphertext_size, + checksum_sha256 = EXCLUDED.checksum_sha256, + uploaded_at = now() + `, transferID, chunk.OpaqueFileID, chunk.ChunkIndex, chunk.ObjectKey, chunk.CiphertextSize, chunk.ChecksumSHA256) + if err != nil { + return fmt.Errorf("upsert chunk %s/%d: %w", chunk.OpaqueFileID, chunk.ChunkIndex, err) + } + + if !slices.Contains(filesTouched, chunk.OpaqueFileID) { + filesTouched = append(filesTouched, chunk.OpaqueFileID) + } + } + + for _, fileID := range filesTouched { + var totalChunks int + if err = tx.QueryRow(ctx, `SELECT total_chunks FROM transfer_files WHERE transfer_id = $1 AND opaque_file_id = $2`, transferID, fileID).Scan(&totalChunks); err != nil { + return fmt.Errorf("select file chunk count %s: %w", fileID, err) + } + + var uploadedCount int + if err = tx.QueryRow(ctx, `SELECT COUNT(*) FROM transfer_chunks WHERE transfer_id = $1 AND opaque_file_id = $2`, transferID, fileID).Scan(&uploadedCount); err != nil { + return fmt.Errorf("count uploaded chunks %s: %w", fileID, err) + } + + status := "pending" + if uploadedCount >= totalChunks { + status = "complete" + } + + if err = tx.Exec(ctx, `UPDATE transfer_files SET upload_status = $3, updated_at = now() WHERE transfer_id = $1 AND opaque_file_id = $2`, transferID, fileID, status); err != nil { + return fmt.Errorf("update file status %s: %w", fileID, err) + } + } + + if err = tx.Exec(ctx, `UPDATE transfers SET updated_at = now() WHERE id = $1`, transferID); err != nil { + return fmt.Errorf("touch transfer: %w", err) + } + + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("commit complete chunks: %w", err) + } + + return nil +} + +// GetResumeState reconstructs the transfer, its files, and uploaded chunk indexes. +func (r *PostgresRepository) GetResumeState(ctx context.Context, transferID string) (models.TransferResumeState, error) { + transfer, err := r.GetTransfer(ctx, transferID) + if err != nil { + return models.TransferResumeState{}, err + } + + files, err := r.ListFiles(ctx, transferID) + if err != nil { + return models.TransferResumeState{}, err + } + + rows, err := r.db.Query(ctx, ` + SELECT opaque_file_id, chunk_index + FROM transfer_chunks + WHERE transfer_id = $1 + ORDER BY opaque_file_id, chunk_index + `, transferID) + if err != nil { + return models.TransferResumeState{}, fmt.Errorf("query chunks: %w", err) + } + defer rows.Close() + + uploaded := map[string][]int{} + for rows.Next() { + var fileID string + var chunkIndex int + if err := rows.Scan(&fileID, &chunkIndex); err != nil { + return models.TransferResumeState{}, fmt.Errorf("scan chunk: %w", err) + } + uploaded[fileID] = append(uploaded[fileID], chunkIndex) + } + if err := rows.Err(); err != nil { + return models.TransferResumeState{}, fmt.Errorf("iterate chunks: %w", err) + } + + return models.TransferResumeState{ + Transfer: transfer, + Files: files, + UploadedChunks: uploaded, + }, nil +} + +// SetManifest stores the manifest object location after the ciphertext upload succeeds. +func (r *PostgresRepository) SetManifest(ctx context.Context, transferID string, objectKey string, ciphertextSize int64) error { + taggedStatus := models.TransferStatusUploading + err := r.db.Exec(ctx, ` + UPDATE transfers + SET manifest_object_key = $2, manifest_ciphertext_size = $3, status = $4, updated_at = now() + WHERE id = $1 + `, transferID, objectKey, ciphertextSize, taggedStatus) + if err != nil { + return fmt.Errorf("set manifest: %w", err) + } + + return nil +} + +// FinalizeTransfer verifies completeness and promotes the transfer to ready state. +func (r *PostgresRepository) FinalizeTransfer(ctx context.Context, transferID string, wrappedRootKey string, totalFiles int, totalCiphertextBytes int64) error { + tx, err := r.db.Begin(ctx) + if err != nil { + return fmt.Errorf("begin finalize: %w", err) + } + defer tx.Rollback(ctx) + + var manifestObjectKey string + if err = tx.QueryRow(ctx, `SELECT manifest_object_key FROM transfers WHERE id = $1`, transferID).Scan(&manifestObjectKey); err != nil { + return fmt.Errorf("select manifest object key: %w", err) + } + if manifestObjectKey == "" { + return fmt.Errorf("manifest not registered") + } + + var incompleteCount int + if err = tx.QueryRow(ctx, ` + SELECT COUNT(*) + FROM transfer_files + WHERE transfer_id = $1 AND upload_status <> 'complete' + `, transferID).Scan(&incompleteCount); err != nil { + return fmt.Errorf("count incomplete files: %w", err) + } + if incompleteCount > 0 { + return fmt.Errorf("upload incomplete") + } + + finalizedAt := time.Now().UTC() + err = tx.Exec(ctx, ` + UPDATE transfers + SET wrapped_root_key = $2, + total_files = $3, + total_ciphertext_bytes = $4, + status = $5, + finalized_at = $6, + updated_at = $6 + WHERE id = $1 + `, transferID, wrappedRootKey, totalFiles, totalCiphertextBytes, models.TransferStatusReady, finalizedAt) + if err != nil { + return fmt.Errorf("update finalized transfer: %w", err) + } + + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("commit finalize: %w", err) + } + + return nil +} + +// UpdateTransfer applies supported metadata changes without rewriting immutable fields. +func (r *PostgresRepository) UpdateTransfer(ctx context.Context, transferID string, params models.UpdateTransferParams) error { + if params.ManifestObjectKey != nil { + err := r.db.Exec(ctx, ` + UPDATE transfers + SET manifest_object_key = $2, updated_at = now() + WHERE id = $1 + `, transferID, *params.ManifestObjectKey) + if err != nil { + return fmt.Errorf("update manifest object key: %w", err) + } + } + + if params.ExpiresAt != nil { + err := r.db.Exec(ctx, `UPDATE transfers SET expires_at = $2, updated_at = now() WHERE id = $1`, transferID, *params.ExpiresAt) + if err != nil { + return fmt.Errorf("update expires_at: %w", err) + } + } + + if params.ManifestCiphertextSize != nil { + err := r.db.Exec(ctx, ` + UPDATE transfers + SET manifest_ciphertext_size = $2, updated_at = now() + WHERE id = $1 + `, transferID, *params.ManifestCiphertextSize) + if err != nil { + return fmt.Errorf("update manifest size: %w", err) + } + } + + return nil +} + +// MarkDeleted tombstones a transfer while retaining enough metadata for cleanup. +func (r *PostgresRepository) MarkDeleted(ctx context.Context, transferID string) error { + err := r.db.Exec(ctx, ` + UPDATE transfers + SET status = $2, deleted_at = now(), updated_at = now() + WHERE id = $1 + `, transferID, models.TransferStatusDeleted) + if err != nil { + return fmt.Errorf("mark deleted: %w", err) + } + + return nil +} + +// ListCleanupCandidates returns transfers whose remote objects should be purged. +func (r *PostgresRepository) ListCleanupCandidates(ctx context.Context, limit int) ([]models.Transfer, error) { + rows, err := r.db.Query(ctx, ` + SELECT + id, status, wrapped_root_key, manifest_object_key, manifest_ciphertext_size, + total_files, total_ciphertext_bytes, expires_at, created_at, updated_at, + finalized_at, manage_token_hash, deleted_at, purged_at + FROM transfers + WHERE purged_at IS NULL + AND ( + (status = 'deleted' AND deleted_at IS NOT NULL) + OR (status IN ('ready', 'uploading', 'draft', 'failed') AND expires_at <= now()) + OR status = 'expired' + ) + ORDER BY updated_at ASC + LIMIT $1 + `, limit) + if err != nil { + return nil, fmt.Errorf("query cleanup candidates: %w", err) + } + defer rows.Close() + + transfers := []models.Transfer{} + for rows.Next() { + var transfer models.Transfer + if err := rows.Scan( + &transfer.ID, + &transfer.Status, + &transfer.WrappedRootKey, + &transfer.ManifestObjectKey, + &transfer.ManifestCiphertextSize, + &transfer.TotalFiles, + &transfer.TotalCiphertextBytes, + &transfer.ExpiresAt, + &transfer.CreatedAt, + &transfer.UpdatedAt, + &transfer.FinalizedAt, + &transfer.ManageTokenHash, + &transfer.DeletedAt, + &transfer.PurgedAt, + ); err != nil { + return nil, fmt.Errorf("scan cleanup transfer: %w", err) + } + if transfer.Status != models.TransferStatusDeleted && transfer.ExpiresAt.Before(time.Now().UTC()) { + transfer.Status = models.TransferStatusExpired + } + transfers = append(transfers, transfer) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate cleanup transfers: %w", err) + } + + return transfers, nil +} + +// MarkPurged records that all remote objects for a transfer have been removed. +func (r *PostgresRepository) MarkPurged(ctx context.Context, transferID string) error { + err := r.db.Exec(ctx, ` + UPDATE transfers + SET status = $2, purged_at = now(), updated_at = now() + WHERE id = $1 + `, transferID, models.TransferStatusExpired) + if err != nil { + return fmt.Errorf("mark purged: %w", err) + } + + return nil +} diff --git a/apps/api/internal/repo/postgres_adapter_integration_test.go b/apps/api/internal/repo/postgres_adapter_integration_test.go new file mode 100644 index 0000000..2699738 --- /dev/null +++ b/apps/api/internal/repo/postgres_adapter_integration_test.go @@ -0,0 +1,56 @@ +package repo + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPGXPoolDBAndPGXTxAdapters(t *testing.T) { + skipIfDockerUnavailable(t) + + ctx := context.Background() + pool := startPostgresTestDB(t, ctx) + database := pgxPoolDB{pool: pool} + + require.NoError(t, database.Exec(ctx, `CREATE TEMP TABLE adapter_test (id text PRIMARY KEY)`)) + + rows, err := database.Query(ctx, `SELECT 'row-value'`) + require.NoError(t, err) + require.True(t, rows.Next()) + var rowValue string + require.NoError(t, rows.Scan(&rowValue)) + require.Equal(t, "row-value", rowValue) + rows.Close() + require.NoError(t, rows.Err()) + + var scalar string + require.NoError(t, database.QueryRow(ctx, `SELECT 'query-row-value'`).Scan(&scalar)) + require.Equal(t, "query-row-value", scalar) + + tx, err := database.Begin(ctx) + require.NoError(t, err) + require.NoError(t, tx.Exec(ctx, `INSERT INTO adapter_test (id) VALUES ($1)`, "committed")) + require.NoError(t, tx.QueryRow(ctx, `SELECT id FROM adapter_test WHERE id = $1`, "committed").Scan(&scalar)) + require.Equal(t, "committed", scalar) + require.NoError(t, tx.Commit(ctx)) + + tx, err = database.Begin(ctx) + require.NoError(t, err) + require.NoError(t, tx.Exec(ctx, `INSERT INTO adapter_test (id) VALUES ($1)`, "rolled-back")) + require.NoError(t, tx.Rollback(ctx)) + + rows, err = database.Query(ctx, `SELECT id FROM adapter_test ORDER BY id`) + require.NoError(t, err) + defer rows.Close() + + values := []string{} + for rows.Next() { + var id string + require.NoError(t, rows.Scan(&id)) + values = append(values, id) + } + require.NoError(t, rows.Err()) + require.Equal(t, []string{"committed"}, values) +} diff --git a/apps/api/internal/repo/postgres_core_unit_test.go b/apps/api/internal/repo/postgres_core_unit_test.go new file mode 100644 index 0000000..efd65fa --- /dev/null +++ b/apps/api/internal/repo/postgres_core_unit_test.go @@ -0,0 +1,526 @@ +package repo + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/models" +) + +func TestPostgresRepositoryTransferAccessors(t *testing.T) { + t.Parallel() + + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Second) + transfer := createTestTransfer("transfer-unit", models.TransferStatusReady, now.Add(time.Hour)) + transfer.WrappedRootKey = `{"key":1}` + transfer.ManifestObjectKey = "transfers/unit/manifest.bin" + transfer.ManifestCiphertextSize = 88 + transfer.TotalFiles = 2 + transfer.TotalCiphertextBytes = 123 + transfer.FinalizedAt = &now + + t.Run("create transfer success and error", func(t *testing.T) { + execCalls := 0 + repository := PostgresRepository{db: stubPostgresDB{ + execFn: func(_ context.Context, _ string, _ ...any) error { + execCalls++ + if execCalls == 2 { + return errors.New("boom") + } + return nil + }, + }} + + require.NoError(t, repository.CreateTransfer(ctx, transfer)) + err := repository.CreateTransfer(ctx, transfer) + require.ErrorContains(t, err, "insert transfer") + }) + + t.Run("get transfer handles not found, query errors, and success", func(t *testing.T) { + call := 0 + repository := PostgresRepository{db: stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + call++ + switch call { + case 1: + return stubRow{err: pgx.ErrNoRows} + case 2: + return stubRow{err: errors.New("boom")} + default: + return stubRow{values: transferScanValues(transfer)} + } + }, + }} + + _, err := repository.GetTransfer(ctx, transfer.ID) + require.ErrorIs(t, err, ErrNotFound) + + _, err = repository.GetTransfer(ctx, transfer.ID) + require.ErrorContains(t, err, "select transfer") + + stored, err := repository.GetTransfer(ctx, transfer.ID) + require.NoError(t, err) + require.Equal(t, transfer.ID, stored.ID) + require.Equal(t, transfer.ManifestObjectKey, stored.ManifestObjectKey) + require.Equal(t, transfer.TotalCiphertextBytes, stored.TotalCiphertextBytes) + require.NotNil(t, stored.FinalizedAt) + }) +} + +func TestPostgresRepositoryRegisterFiles(t *testing.T) { + t.Parallel() + + ctx := context.Background() + files := []models.TransferFile{ + {OpaqueFileID: "file-a", TotalChunks: 2, CiphertextBytes: 64, PlaintextBytes: int64ptr(32), ChunkSize: 32}, + {OpaqueFileID: "file-b", TotalChunks: 1, CiphertextBytes: 16, PlaintextBytes: int64ptr(8), ChunkSize: 16}, + } + + t.Run("success commits after inserting files", func(t *testing.T) { + execCalls := 0 + tx := &stubPostgresTx{ + execFn: func(context.Context, string, ...any) error { + execCalls++ + return nil + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + require.NoError(t, repository.RegisterFiles(ctx, "transfer-1", files)) + require.Equal(t, 3, execCalls) + require.True(t, tx.commitCalled) + }) + + t.Run("begin failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return nil, errors.New("boom") + }, + }} + + err := repository.RegisterFiles(ctx, "transfer-1", files) + require.ErrorContains(t, err, "begin register files") + }) + + t.Run("file insert failure", func(t *testing.T) { + tx := &stubPostgresTx{ + execFn: func(_ context.Context, sql string, args ...any) error { + if strings.Contains(sql, "INSERT INTO transfer_files") && args[1] == "file-a" { + return errors.New("boom") + } + return nil + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.RegisterFiles(ctx, "transfer-1", files) + require.ErrorContains(t, err, "insert file file-a") + }) + + t.Run("status update failure", func(t *testing.T) { + tx := &stubPostgresTx{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "UPDATE transfers SET status") { + return errors.New("boom") + } + return nil + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.RegisterFiles(ctx, "transfer-1", files) + require.ErrorContains(t, err, "set uploading status") + }) + + t.Run("commit failure", func(t *testing.T) { + tx := &stubPostgresTx{commitErr: errors.New("boom")} + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.RegisterFiles(ctx, "transfer-1", files) + require.ErrorContains(t, err, "commit register files") + }) +} + +func TestPostgresRepositoryListFiles(t *testing.T) { + t.Parallel() + + ctx := context.Background() + file := models.TransferFile{ + TransferID: "transfer-1", + OpaqueFileID: "file-a", + TotalChunks: 2, + CiphertextBytes: 64, + PlaintextBytes: int64ptr(32), + ChunkSize: 32, + UploadStatus: "pending", + CreatedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + } + + t.Run("query failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + queryFn: func(context.Context, string, ...any) (postgresRows, error) { + return nil, errors.New("boom") + }, + }} + + _, err := repository.ListFiles(ctx, "transfer-1") + require.ErrorContains(t, err, "query files") + }) + + t.Run("scan failure", func(t *testing.T) { + rows := &stubRows{values: [][]any{fileScanValues(file)}, scanErrAt: 1, scanErr: errors.New("boom")} + repository := PostgresRepository{db: stubPostgresDB{ + queryFn: func(context.Context, string, ...any) (postgresRows, error) { + return rows, nil + }, + }} + + _, err := repository.ListFiles(ctx, "transfer-1") + require.ErrorContains(t, err, "scan file") + require.True(t, rows.closed) + }) + + t.Run("iteration failure", func(t *testing.T) { + rows := &stubRows{values: [][]any{fileScanValues(file)}, err: errors.New("boom")} + repository := PostgresRepository{db: stubPostgresDB{ + queryFn: func(context.Context, string, ...any) (postgresRows, error) { + return rows, nil + }, + }} + + _, err := repository.ListFiles(ctx, "transfer-1") + require.ErrorContains(t, err, "iterate files") + }) + + t.Run("success", func(t *testing.T) { + rows := &stubRows{values: [][]any{fileScanValues(file)}} + repository := PostgresRepository{db: stubPostgresDB{ + queryFn: func(context.Context, string, ...any) (postgresRows, error) { + return rows, nil + }, + }} + + files, err := repository.ListFiles(ctx, "transfer-1") + require.NoError(t, err) + require.Len(t, files, 1) + require.Equal(t, "file-a", files[0].OpaqueFileID) + }) +} + +func TestPostgresRepositoryCompleteChunks(t *testing.T) { + t.Parallel() + + ctx := context.Background() + chunks := []models.TransferChunk{ + {OpaqueFileID: "file-a", ChunkIndex: 0, ObjectKey: "chunk-0", CiphertextSize: 10, ChecksumSHA256: "sum-0"}, + {OpaqueFileID: "file-a", ChunkIndex: 1, ObjectKey: "chunk-1", CiphertextSize: 10, ChecksumSHA256: "sum-1"}, + } + + t.Run("success marks completed files once", func(t *testing.T) { + totalQueries := 0 + countQueries := 0 + statusUpdates := 0 + tx := &stubPostgresTx{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "UPDATE transfer_files SET upload_status") { + statusUpdates++ + } + return nil + }, + queryRowFn: func(_ context.Context, sql string, _ ...any) postgresRow { + switch { + case strings.Contains(sql, "SELECT total_chunks"): + totalQueries++ + return stubRow{values: []any{2}} + case strings.Contains(sql, "SELECT COUNT(*) FROM transfer_chunks"): + countQueries++ + return stubRow{values: []any{2}} + default: + return stubRow{err: fmt.Errorf("unexpected query: %s", sql)} + } + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + require.NoError(t, repository.CompleteChunks(ctx, "transfer-1", chunks)) + require.Equal(t, 1, totalQueries) + require.Equal(t, 1, countQueries) + require.Equal(t, 1, statusUpdates) + require.True(t, tx.commitCalled) + }) + + t.Run("begin failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return nil, errors.New("boom") + }, + }} + + err := repository.CompleteChunks(ctx, "transfer-1", chunks) + require.ErrorContains(t, err, "begin complete chunks") + }) + + t.Run("chunk upsert failure", func(t *testing.T) { + tx := &stubPostgresTx{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "INSERT INTO transfer_chunks") { + return errors.New("boom") + } + return nil + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.CompleteChunks(ctx, "transfer-1", chunks) + require.ErrorContains(t, err, "upsert chunk file-a/0") + }) + + t.Run("select total chunks failure", func(t *testing.T) { + tx := &stubPostgresTx{ + queryRowFn: func(_ context.Context, sql string, _ ...any) postgresRow { + if strings.Contains(sql, "SELECT total_chunks") { + return stubRow{err: errors.New("boom")} + } + return stubRow{values: []any{1}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.CompleteChunks(ctx, "transfer-1", chunks[:1]) + require.ErrorContains(t, err, "select file chunk count file-a") + }) + + t.Run("count uploaded chunks failure", func(t *testing.T) { + tx := &stubPostgresTx{ + queryRowFn: func(_ context.Context, sql string, _ ...any) postgresRow { + if strings.Contains(sql, "SELECT COUNT(*) FROM transfer_chunks") { + return stubRow{err: errors.New("boom")} + } + return stubRow{values: []any{1}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.CompleteChunks(ctx, "transfer-1", chunks[:1]) + require.ErrorContains(t, err, "count uploaded chunks file-a") + }) + + t.Run("update file status failure", func(t *testing.T) { + tx := &stubPostgresTx{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "UPDATE transfer_files SET upload_status") { + return errors.New("boom") + } + return nil + }, + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{1}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.CompleteChunks(ctx, "transfer-1", chunks[:1]) + require.ErrorContains(t, err, "update file status file-a") + }) + + t.Run("touch transfer failure", func(t *testing.T) { + tx := &stubPostgresTx{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "UPDATE transfers SET updated_at") { + return errors.New("boom") + } + return nil + }, + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{1}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.CompleteChunks(ctx, "transfer-1", chunks[:1]) + require.ErrorContains(t, err, "touch transfer") + }) + + t.Run("commit failure", func(t *testing.T) { + tx := &stubPostgresTx{ + commitErr: errors.New("boom"), + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{1}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.CompleteChunks(ctx, "transfer-1", chunks[:1]) + require.ErrorContains(t, err, "commit complete chunks") + }) +} + +func TestPostgresRepositoryGetResumeState(t *testing.T) { + t.Parallel() + + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Second) + transfer := createTestTransfer("transfer-resume", models.TransferStatusUploading, now.Add(time.Hour)) + file := models.TransferFile{ + TransferID: transfer.ID, + OpaqueFileID: "file-a", + TotalChunks: 2, + CiphertextBytes: 64, + PlaintextBytes: int64ptr(32), + ChunkSize: 32, + UploadStatus: "pending", + CreatedAt: now, + UpdatedAt: now, + } + + t.Run("transfer lookup failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{err: pgx.ErrNoRows} + }, + }} + + _, err := repository.GetResumeState(ctx, transfer.ID) + require.ErrorIs(t, err, ErrNotFound) + }) + + t.Run("list files failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: transferScanValues(transfer)} + }, + queryFn: func(_ context.Context, sql string, _ ...any) (postgresRows, error) { + if strings.Contains(sql, "FROM transfer_files") { + return nil, errors.New("boom") + } + return &stubRows{}, nil + }, + }} + + _, err := repository.GetResumeState(ctx, transfer.ID) + require.ErrorContains(t, err, "query files") + }) + + t.Run("query chunks failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: transferScanValues(transfer)} + }, + queryFn: func(_ context.Context, sql string, _ ...any) (postgresRows, error) { + if strings.Contains(sql, "FROM transfer_files") { + return &stubRows{values: [][]any{fileScanValues(file)}}, nil + } + return nil, errors.New("boom") + }, + }} + + _, err := repository.GetResumeState(ctx, transfer.ID) + require.ErrorContains(t, err, "query chunks") + }) + + t.Run("scan chunk failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: transferScanValues(transfer)} + }, + queryFn: func(_ context.Context, sql string, _ ...any) (postgresRows, error) { + if strings.Contains(sql, "FROM transfer_files") { + return &stubRows{values: [][]any{fileScanValues(file)}}, nil + } + return &stubRows{values: [][]any{{"file-a", 0}}, scanErrAt: 1, scanErr: errors.New("boom")}, nil + }, + }} + + _, err := repository.GetResumeState(ctx, transfer.ID) + require.ErrorContains(t, err, "scan chunk") + }) + + t.Run("iterate chunks failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: transferScanValues(transfer)} + }, + queryFn: func(_ context.Context, sql string, _ ...any) (postgresRows, error) { + if strings.Contains(sql, "FROM transfer_files") { + return &stubRows{values: [][]any{fileScanValues(file)}}, nil + } + return &stubRows{values: [][]any{{"file-a", 0}}, err: errors.New("boom")}, nil + }, + }} + + _, err := repository.GetResumeState(ctx, transfer.ID) + require.ErrorContains(t, err, "iterate chunks") + }) + + t.Run("success", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: transferScanValues(transfer)} + }, + queryFn: func(_ context.Context, sql string, _ ...any) (postgresRows, error) { + if strings.Contains(sql, "FROM transfer_files") { + return &stubRows{values: [][]any{fileScanValues(file)}}, nil + } + return &stubRows{values: [][]any{{"file-a", 0}, {"file-a", 1}}}, nil + }, + }} + + state, err := repository.GetResumeState(ctx, transfer.ID) + require.NoError(t, err) + require.Equal(t, transfer.ID, state.Transfer.ID) + require.Len(t, state.Files, 1) + require.Equal(t, []int{0, 1}, state.UploadedChunks["file-a"]) + }) +} diff --git a/apps/api/internal/repo/postgres_integration_test.go b/apps/api/internal/repo/postgres_integration_test.go new file mode 100644 index 0000000..3a43d2d --- /dev/null +++ b/apps/api/internal/repo/postgres_integration_test.go @@ -0,0 +1,272 @@ +package repo + +import ( + "context" + "os/exec" + "runtime" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/xdrop/monorepo/internal/models" +) + +func TestRunMigrationsAndRepositoryLifecycle(t *testing.T) { + skipIfDockerUnavailable(t) + + ctx := context.Background() + db := startPostgresTestDB(t, ctx) + require.NoError(t, RunMigrations(ctx, db)) + require.NoError(t, RunMigrations(ctx, db)) + + repository := NewPostgresRepository(db) + + _, err := repository.GetTransfer(ctx, "missing-transfer") + require.ErrorIs(t, err, ErrNotFound) + + now := time.Now().UTC().Truncate(time.Second) + transfer := models.Transfer{ + ID: "transfer-1", + Status: models.TransferStatusDraft, + ExpiresAt: now.Add(time.Hour), + CreatedAt: now, + UpdatedAt: now, + ManageTokenHash: "hash-1", + } + require.NoError(t, repository.CreateTransfer(ctx, transfer)) + + files := []models.TransferFile{ + { + TransferID: transfer.ID, + OpaqueFileID: "file-a", + TotalChunks: 2, + CiphertextBytes: 64, + PlaintextBytes: int64ptr(32), + ChunkSize: 32, + }, + { + TransferID: transfer.ID, + OpaqueFileID: "file-b", + TotalChunks: 1, + CiphertextBytes: 48, + PlaintextBytes: int64ptr(24), + ChunkSize: 48, + }, + } + require.NoError(t, repository.RegisterFiles(ctx, transfer.ID, files)) + + stored, err := repository.GetTransfer(ctx, transfer.ID) + require.NoError(t, err) + require.Equal(t, models.TransferStatusUploading, stored.Status) + + listedFiles, err := repository.ListFiles(ctx, transfer.ID) + require.NoError(t, err) + require.Len(t, listedFiles, 2) + require.Equal(t, "file-a", listedFiles[0].OpaqueFileID) + require.Equal(t, "pending", listedFiles[0].UploadStatus) + require.Equal(t, "file-b", listedFiles[1].OpaqueFileID) + + require.NoError(t, repository.CompleteChunks(ctx, transfer.ID, []models.TransferChunk{ + { + TransferID: transfer.ID, + OpaqueFileID: "file-a", + ChunkIndex: 0, + ObjectKey: "transfers/transfer-1/files/file-a/chunks/00000000.bin", + CiphertextSize: 32, + ChecksumSHA256: "checksum-a0", + }, + { + TransferID: transfer.ID, + OpaqueFileID: "file-a", + ChunkIndex: 0, + ObjectKey: "transfers/transfer-1/files/file-a/chunks/00000000.bin", + CiphertextSize: 32, + ChecksumSHA256: "checksum-a0-updated", + }, + { + TransferID: transfer.ID, + OpaqueFileID: "file-b", + ChunkIndex: 0, + ObjectKey: "transfers/transfer-1/files/file-b/chunks/00000000.bin", + CiphertextSize: 48, + ChecksumSHA256: "checksum-b0", + }, + })) + + resume, err := repository.GetResumeState(ctx, transfer.ID) + require.NoError(t, err) + require.Equal(t, []int{0}, resume.UploadedChunks["file-a"]) + require.Equal(t, []int{0}, resume.UploadedChunks["file-b"]) + + listedFiles, err = repository.ListFiles(ctx, transfer.ID) + require.NoError(t, err) + require.Equal(t, "pending", listedFiles[0].UploadStatus) + require.Equal(t, "complete", listedFiles[1].UploadStatus) + + require.NoError(t, repository.CompleteChunks(ctx, transfer.ID, []models.TransferChunk{ + { + TransferID: transfer.ID, + OpaqueFileID: "file-a", + ChunkIndex: 1, + ObjectKey: "transfers/transfer-1/files/file-a/chunks/00000001.bin", + CiphertextSize: 32, + ChecksumSHA256: "checksum-a1", + }, + })) + require.NoError(t, repository.SetManifest(ctx, transfer.ID, "transfers/transfer-1/manifest.bin", 77)) + require.NoError(t, repository.FinalizeTransfer(ctx, transfer.ID, `{"version":1}`, 2, 112)) + + finalized, err := repository.GetTransfer(ctx, transfer.ID) + require.NoError(t, err) + require.Equal(t, models.TransferStatusReady, finalized.Status) + require.Equal(t, "transfers/transfer-1/manifest.bin", finalized.ManifestObjectKey) + require.Equal(t, `{"version":1}`, finalized.WrappedRootKey) + require.Equal(t, 2, finalized.TotalFiles) + require.Equal(t, int64(112), finalized.TotalCiphertextBytes) + require.NotNil(t, finalized.FinalizedAt) + + newExpiry := now.Add(3 * time.Hour) + newManifestKey := "transfers/transfer-1/manifest-renamed.bin" + newManifestSize := int64(99) + require.NoError(t, repository.UpdateTransfer(ctx, transfer.ID, models.UpdateTransferParams{ + ExpiresAt: &newExpiry, + ManifestObjectKey: &newManifestKey, + ManifestCiphertextSize: &newManifestSize, + })) + + updated, err := repository.GetTransfer(ctx, transfer.ID) + require.NoError(t, err) + require.WithinDuration(t, newExpiry, updated.ExpiresAt, time.Second) + require.Equal(t, newManifestKey, updated.ManifestObjectKey) + require.Equal(t, newManifestSize, updated.ManifestCiphertextSize) +} + +func TestPostgresRepositoryFinalizeTransferValidatesManifestAndUploads(t *testing.T) { + skipIfDockerUnavailable(t) + + ctx := context.Background() + db := startPostgresTestDB(t, ctx) + require.NoError(t, RunMigrations(ctx, db)) + + repository := NewPostgresRepository(db) + transfer := createTestTransfer("transfer-guard", models.TransferStatusDraft, time.Now().UTC().Add(time.Hour)) + require.NoError(t, repository.CreateTransfer(ctx, transfer)) + require.NoError(t, repository.RegisterFiles(ctx, transfer.ID, []models.TransferFile{{ + TransferID: transfer.ID, + OpaqueFileID: "file-a", + TotalChunks: 1, + CiphertextBytes: 16, + ChunkSize: 16, + }})) + + err := repository.FinalizeTransfer(ctx, transfer.ID, `{"version":1}`, 1, 16) + require.ErrorContains(t, err, "manifest not registered") + + require.NoError(t, repository.SetManifest(ctx, transfer.ID, "transfers/transfer-guard/manifest.bin", 32)) + err = repository.FinalizeTransfer(ctx, transfer.ID, `{"version":1}`, 1, 16) + require.ErrorContains(t, err, "upload incomplete") +} + +func TestPostgresRepositoryCleanupCandidatesAndPurging(t *testing.T) { + skipIfDockerUnavailable(t) + + ctx := context.Background() + db := startPostgresTestDB(t, ctx) + require.NoError(t, RunMigrations(ctx, db)) + + repository := NewPostgresRepository(db) + now := time.Now().UTC() + + expiredReady := createTestTransfer("expired-ready", models.TransferStatusReady, now.Add(-time.Hour)) + deletedTransfer := createTestTransfer("deleted-transfer", models.TransferStatusReady, now.Add(time.Hour)) + futureTransfer := createTestTransfer("future-transfer", models.TransferStatusUploading, now.Add(time.Hour)) + purgedTransfer := createTestTransfer("purged-transfer", models.TransferStatusReady, now.Add(-2*time.Hour)) + + for _, transfer := range []models.Transfer{expiredReady, deletedTransfer, futureTransfer, purgedTransfer} { + require.NoError(t, repository.CreateTransfer(ctx, transfer)) + } + require.NoError(t, repository.MarkDeleted(ctx, deletedTransfer.ID)) + require.NoError(t, repository.MarkPurged(ctx, purgedTransfer.ID)) + + candidates, err := repository.ListCleanupCandidates(ctx, 10) + require.NoError(t, err) + + candidateByID := map[string]models.Transfer{} + for _, candidate := range candidates { + candidateByID[candidate.ID] = candidate + } + + require.Contains(t, candidateByID, expiredReady.ID) + require.Contains(t, candidateByID, deletedTransfer.ID) + require.NotContains(t, candidateByID, futureTransfer.ID) + require.NotContains(t, candidateByID, purgedTransfer.ID) + require.Equal(t, models.TransferStatusExpired, candidateByID[expiredReady.ID].Status) + require.Equal(t, models.TransferStatusDeleted, candidateByID[deletedTransfer.ID].Status) + + require.NoError(t, repository.MarkPurged(ctx, expiredReady.ID)) + purged, err := repository.GetTransfer(ctx, expiredReady.ID) + require.NoError(t, err) + require.Equal(t, models.TransferStatusExpired, purged.Status) + require.NotNil(t, purged.PurgedAt) +} + +func startPostgresTestDB(t *testing.T, ctx context.Context) *pgxpool.Pool { + t.Helper() + + container, err := tcpostgres.Run( + ctx, + "postgres:16-alpine", + tcpostgres.WithDatabase("xdrop"), + tcpostgres.WithUsername("xdrop"), + tcpostgres.WithPassword("xdrop"), + tcpostgres.BasicWaitStrategies(), + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, testcontainers.TerminateContainer(container)) + }) + + connectionString, err := container.ConnectionString(ctx, "sslmode=disable") + require.NoError(t, err) + + db, err := pgxpool.New(ctx, connectionString) + require.NoError(t, err) + require.NoError(t, db.Ping(ctx)) + t.Cleanup(db.Close) + + return db +} + +func createTestTransfer(id string, status models.TransferStatus, expiresAt time.Time) models.Transfer { + now := time.Now().UTC().Truncate(time.Second) + return models.Transfer{ + ID: id, + Status: status, + ExpiresAt: expiresAt, + CreatedAt: now, + UpdatedAt: now, + ManageTokenHash: "manage-token-hash", + } +} + +func int64ptr(value int64) *int64 { + return &value +} + +func skipIfDockerUnavailable(t *testing.T) { + t.Helper() + + if testing.Short() { + t.Skip("skipping docker-backed integration test in short mode") + } + if runtime.GOOS == "windows" { + t.Skip("skipping docker-backed integration test on windows") + } + + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skipf("skipping docker-backed integration test: %v", err) + } +} diff --git a/apps/api/internal/repo/postgres_mutation_unit_test.go b/apps/api/internal/repo/postgres_mutation_unit_test.go new file mode 100644 index 0000000..5d55fff --- /dev/null +++ b/apps/api/internal/repo/postgres_mutation_unit_test.go @@ -0,0 +1,321 @@ +package repo + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/models" +) + +func TestPostgresRepositoryFinalizeUpdateAndCleanup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + now := time.Now().UTC().Truncate(time.Second) + + t.Run("set manifest success and error", func(t *testing.T) { + execCalls := 0 + repository := PostgresRepository{db: stubPostgresDB{ + execFn: func(context.Context, string, ...any) error { + execCalls++ + if execCalls == 2 { + return errors.New("boom") + } + return nil + }, + }} + + require.NoError(t, repository.SetManifest(ctx, "transfer-1", "manifest.bin", 88)) + err := repository.SetManifest(ctx, "transfer-1", "manifest.bin", 88) + require.ErrorContains(t, err, "set manifest") + }) + + t.Run("finalize variants", func(t *testing.T) { + t.Run("begin failure", func(t *testing.T) { + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return nil, errors.New("boom") + }, + }} + + err := repository.FinalizeTransfer(ctx, "transfer-1", "root", 1, 10) + require.ErrorContains(t, err, "begin finalize") + }) + + t.Run("manifest query failure", func(t *testing.T) { + tx := &stubPostgresTx{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{err: errors.New("boom")} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.FinalizeTransfer(ctx, "transfer-1", "root", 1, 10) + require.ErrorContains(t, err, "select manifest object key") + }) + + t.Run("requires manifest", func(t *testing.T) { + tx := &stubPostgresTx{ + queryRowFn: func(context.Context, string, ...any) postgresRow { + return stubRow{values: []any{""}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.FinalizeTransfer(ctx, "transfer-1", "root", 1, 10) + require.ErrorContains(t, err, "manifest not registered") + }) + + t.Run("incomplete count failure", func(t *testing.T) { + tx := &stubPostgresTx{ + queryRowFn: func(_ context.Context, sql string, _ ...any) postgresRow { + if strings.Contains(sql, "SELECT COUNT(*)") { + return stubRow{err: errors.New("boom")} + } + return stubRow{values: []any{"manifest.bin"}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.FinalizeTransfer(ctx, "transfer-1", "root", 1, 10) + require.ErrorContains(t, err, "count incomplete files") + }) + + t.Run("rejects incomplete uploads", func(t *testing.T) { + tx := &stubPostgresTx{ + queryRowFn: func(_ context.Context, sql string, _ ...any) postgresRow { + if strings.Contains(sql, "SELECT COUNT(*)") { + return stubRow{values: []any{1}} + } + return stubRow{values: []any{"manifest.bin"}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.FinalizeTransfer(ctx, "transfer-1", "root", 1, 10) + require.ErrorContains(t, err, "upload incomplete") + }) + + t.Run("update failure", func(t *testing.T) { + tx := &stubPostgresTx{ + execFn: func(context.Context, string, ...any) error { + return errors.New("boom") + }, + queryRowFn: func(_ context.Context, sql string, _ ...any) postgresRow { + if strings.Contains(sql, "SELECT COUNT(*)") { + return stubRow{values: []any{0}} + } + return stubRow{values: []any{"manifest.bin"}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.FinalizeTransfer(ctx, "transfer-1", "root", 1, 10) + require.ErrorContains(t, err, "update finalized transfer") + }) + + t.Run("commit failure", func(t *testing.T) { + tx := &stubPostgresTx{ + commitErr: errors.New("boom"), + queryRowFn: func(_ context.Context, sql string, _ ...any) postgresRow { + if strings.Contains(sql, "SELECT COUNT(*)") { + return stubRow{values: []any{0}} + } + return stubRow{values: []any{"manifest.bin"}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + err := repository.FinalizeTransfer(ctx, "transfer-1", "root", 1, 10) + require.ErrorContains(t, err, "commit finalize") + }) + + t.Run("success", func(t *testing.T) { + tx := &stubPostgresTx{ + queryRowFn: func(_ context.Context, sql string, _ ...any) postgresRow { + if strings.Contains(sql, "SELECT COUNT(*)") { + return stubRow{values: []any{0}} + } + return stubRow{values: []any{"manifest.bin"}} + }, + } + repository := PostgresRepository{db: stubPostgresDB{ + beginFn: func(context.Context) (postgresTx, error) { + return tx, nil + }, + }} + + require.NoError(t, repository.FinalizeTransfer(ctx, "transfer-1", "root", 1, 10)) + require.True(t, tx.commitCalled) + }) + }) + + t.Run("update transfer covers each field and errors", func(t *testing.T) { + manifestKey := "manifest-renamed.bin" + expiresAt := now.Add(2 * time.Hour) + manifestSize := int64(101) + repository := PostgresRepository{db: stubPostgresDB{ + execFn: func(_ context.Context, _ string, _ ...any) error { + return nil + }, + }} + + require.NoError(t, repository.UpdateTransfer(ctx, "transfer-1", models.UpdateTransferParams{ + ManifestObjectKey: &manifestKey, + ManifestCiphertextSize: &manifestSize, + })) + + err := repository.UpdateTransfer(ctx, "transfer-1", models.UpdateTransferParams{ + ExpiresAt: &expiresAt, + }) + require.NoError(t, err) + + err = repository.UpdateTransfer(ctx, "transfer-1", models.UpdateTransferParams{ + ManifestCiphertextSize: &manifestSize, + }) + require.NoError(t, err) + + repository = PostgresRepository{db: stubPostgresDB{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "manifest_object_key") { + return errors.New("boom") + } + return nil + }, + }} + + err = repository.UpdateTransfer(ctx, "transfer-1", models.UpdateTransferParams{ + ManifestObjectKey: &manifestKey, + }) + require.ErrorContains(t, err, "update manifest object key") + + repository = PostgresRepository{db: stubPostgresDB{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "expires_at") { + return errors.New("boom") + } + return nil + }, + }} + + err = repository.UpdateTransfer(ctx, "transfer-1", models.UpdateTransferParams{ + ExpiresAt: &expiresAt, + }) + require.ErrorContains(t, err, "update expires_at") + + repository = PostgresRepository{db: stubPostgresDB{ + execFn: func(_ context.Context, sql string, _ ...any) error { + if strings.Contains(sql, "manifest_ciphertext_size") { + return errors.New("boom") + } + return nil + }, + }} + + err = repository.UpdateTransfer(ctx, "transfer-1", models.UpdateTransferParams{ + ManifestCiphertextSize: &manifestSize, + }) + require.ErrorContains(t, err, "update manifest size") + + noOpCalls := 0 + repository = PostgresRepository{db: stubPostgresDB{ + execFn: func(context.Context, string, ...any) error { + noOpCalls++ + return nil + }, + }} + require.NoError(t, repository.UpdateTransfer(ctx, "transfer-1", models.UpdateTransferParams{})) + require.Zero(t, noOpCalls) + }) + + t.Run("mark deleted and purged success and error", func(t *testing.T) { + execCalls := 0 + repository := PostgresRepository{db: stubPostgresDB{ + execFn: func(context.Context, string, ...any) error { + execCalls++ + if execCalls == 2 || execCalls == 4 { + return errors.New("boom") + } + return nil + }, + }} + + require.NoError(t, repository.MarkDeleted(ctx, "transfer-1")) + err := repository.MarkDeleted(ctx, "transfer-1") + require.ErrorContains(t, err, "mark deleted") + + require.NoError(t, repository.MarkPurged(ctx, "transfer-1")) + err = repository.MarkPurged(ctx, "transfer-1") + require.ErrorContains(t, err, "mark purged") + }) + + t.Run("list cleanup candidates handles query, scan, iteration, and success", func(t *testing.T) { + expired := createTestTransfer("expired", models.TransferStatusReady, now.Add(-time.Hour)) + deleted := createTestTransfer("deleted", models.TransferStatusDeleted, now.Add(time.Hour)) + deletedAt := now.Add(-30 * time.Minute) + deleted.DeletedAt = &deletedAt + + repository := PostgresRepository{db: stubPostgresDB{ + queryFn: func(context.Context, string, ...any) (postgresRows, error) { + return nil, errors.New("boom") + }, + }} + _, err := repository.ListCleanupCandidates(ctx, 10) + require.ErrorContains(t, err, "query cleanup candidates") + + repository = PostgresRepository{db: stubPostgresDB{ + queryFn: func(context.Context, string, ...any) (postgresRows, error) { + return &stubRows{values: [][]any{transferScanValues(expired)}, scanErrAt: 1, scanErr: errors.New("boom")}, nil + }, + }} + _, err = repository.ListCleanupCandidates(ctx, 10) + require.ErrorContains(t, err, "scan cleanup transfer") + + repository = PostgresRepository{db: stubPostgresDB{ + queryFn: func(context.Context, string, ...any) (postgresRows, error) { + return &stubRows{values: [][]any{transferScanValues(expired)}, err: errors.New("boom")}, nil + }, + }} + _, err = repository.ListCleanupCandidates(ctx, 10) + require.ErrorContains(t, err, "iterate cleanup transfers") + + repository = PostgresRepository{db: stubPostgresDB{ + queryFn: func(context.Context, string, ...any) (postgresRows, error) { + return &stubRows{values: [][]any{transferScanValues(expired), transferScanValues(deleted)}}, nil + }, + }} + candidates, err := repository.ListCleanupCandidates(ctx, 10) + require.NoError(t, err) + require.Len(t, candidates, 2) + require.Equal(t, models.TransferStatusExpired, candidates[0].Status) + require.Equal(t, models.TransferStatusDeleted, candidates[1].Status) + }) +} diff --git a/apps/api/internal/repo/postgres_wrapper_test.go b/apps/api/internal/repo/postgres_wrapper_test.go new file mode 100644 index 0000000..de0fbb3 --- /dev/null +++ b/apps/api/internal/repo/postgres_wrapper_test.go @@ -0,0 +1,17 @@ +package repo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewPostgresRepositoryWrapsPoolBackedDB(t *testing.T) { + t.Parallel() + + repository := NewPostgresRepository(nil) + wrapped, ok := repository.db.(pgxPoolDB) + + require.True(t, ok) + require.Nil(t, wrapped.pool) +} diff --git a/apps/api/internal/repo/repository.go b/apps/api/internal/repo/repository.go new file mode 100644 index 0000000..99a9fff --- /dev/null +++ b/apps/api/internal/repo/repository.go @@ -0,0 +1,23 @@ +package repo + +import ( + "context" + + "github.com/xdrop/monorepo/internal/models" +) + +// Repository defines the persistence contract for transfer lifecycle operations. +type Repository interface { + CreateTransfer(ctx context.Context, transfer models.Transfer) error + GetTransfer(ctx context.Context, transferID string) (models.Transfer, error) + RegisterFiles(ctx context.Context, transferID string, files []models.TransferFile) error + ListFiles(ctx context.Context, transferID string) ([]models.TransferFile, error) + CompleteChunks(ctx context.Context, transferID string, chunks []models.TransferChunk) error + GetResumeState(ctx context.Context, transferID string) (models.TransferResumeState, error) + SetManifest(ctx context.Context, transferID string, objectKey string, ciphertextSize int64) error + FinalizeTransfer(ctx context.Context, transferID string, wrappedRootKey string, totalFiles int, totalCiphertextBytes int64) error + UpdateTransfer(ctx context.Context, transferID string, params models.UpdateTransferParams) error + MarkDeleted(ctx context.Context, transferID string) error + ListCleanupCandidates(ctx context.Context, limit int) ([]models.Transfer, error) + MarkPurged(ctx context.Context, transferID string) error +} diff --git a/apps/api/internal/repo/test_helpers_test.go b/apps/api/internal/repo/test_helpers_test.go new file mode 100644 index 0000000..d193c2f --- /dev/null +++ b/apps/api/internal/repo/test_helpers_test.go @@ -0,0 +1,205 @@ +package repo + +import ( + "context" + "fmt" + "io/fs" + "reflect" + + "github.com/xdrop/monorepo/internal/models" +) + +type stubMigrationFiles struct { + readDirFn func(name string) ([]fs.DirEntry, error) + readFileFn func(name string) ([]byte, error) +} + +func (f stubMigrationFiles) ReadDir(name string) ([]fs.DirEntry, error) { + if f.readDirFn != nil { + return f.readDirFn(name) + } + return nil, nil +} + +func (f stubMigrationFiles) ReadFile(name string) ([]byte, error) { + if f.readFileFn != nil { + return f.readFileFn(name) + } + return nil, nil +} + +type stubPostgresDB struct { + beginFn func(ctx context.Context) (postgresTx, error) + execFn func(ctx context.Context, sql string, args ...any) error + queryFn func(ctx context.Context, sql string, args ...any) (postgresRows, error) + queryRowFn func(ctx context.Context, sql string, args ...any) postgresRow +} + +func (db stubPostgresDB) Begin(ctx context.Context) (postgresTx, error) { + if db.beginFn != nil { + return db.beginFn(ctx) + } + return &stubPostgresTx{}, nil +} + +func (db stubPostgresDB) Exec(ctx context.Context, sql string, args ...any) error { + if db.execFn != nil { + return db.execFn(ctx, sql, args...) + } + return nil +} + +func (db stubPostgresDB) Query(ctx context.Context, sql string, args ...any) (postgresRows, error) { + if db.queryFn != nil { + return db.queryFn(ctx, sql, args...) + } + return &stubRows{}, nil +} + +func (db stubPostgresDB) QueryRow(ctx context.Context, sql string, args ...any) postgresRow { + if db.queryRowFn != nil { + return db.queryRowFn(ctx, sql, args...) + } + return stubRow{} +} + +type stubPostgresTx struct { + commitCalled bool + commitErr error + execFn func(ctx context.Context, sql string, args ...any) error + queryRowFn func(ctx context.Context, sql string, args ...any) postgresRow + rollbackCalled bool + rollbackErr error +} + +func (tx *stubPostgresTx) Commit(context.Context) error { + tx.commitCalled = true + return tx.commitErr +} + +func (tx *stubPostgresTx) Exec(ctx context.Context, sql string, args ...any) error { + if tx.execFn != nil { + return tx.execFn(ctx, sql, args...) + } + return nil +} + +func (tx *stubPostgresTx) QueryRow(ctx context.Context, sql string, args ...any) postgresRow { + if tx.queryRowFn != nil { + return tx.queryRowFn(ctx, sql, args...) + } + return stubRow{} +} + +func (tx *stubPostgresTx) Rollback(context.Context) error { + tx.rollbackCalled = true + return tx.rollbackErr +} + +type stubRow struct { + err error + values []any +} + +func (r stubRow) Scan(dest ...any) error { + if r.err != nil { + return r.err + } + return assignScanValues(dest, r.values) +} + +type stubRows struct { + closed bool + err error + index int + scanErr error + scanErrAt int + values [][]any +} + +func (r *stubRows) Close() { + r.closed = true +} + +func (r *stubRows) Err() error { + return r.err +} + +func (r *stubRows) Next() bool { + if r.index >= len(r.values) { + return false + } + r.index++ + return true +} + +func (r *stubRows) Scan(dest ...any) error { + if r.scanErr != nil && r.index == r.scanErrAt { + return r.scanErr + } + return assignScanValues(dest, r.values[r.index-1]) +} + +func assignScanValues(dest []any, values []any) error { + if len(dest) != len(values) { + return fmt.Errorf("scan value count mismatch: %d != %d", len(dest), len(values)) + } + + for i := range dest { + target := reflect.ValueOf(dest[i]) + if !target.IsValid() || target.Kind() != reflect.Pointer || target.IsNil() { + return fmt.Errorf("scan target %d is not a pointer", i) + } + + elem := target.Elem() + if values[i] == nil { + elem.Set(reflect.Zero(elem.Type())) + continue + } + + value := reflect.ValueOf(values[i]) + switch { + case value.Type().AssignableTo(elem.Type()): + elem.Set(value) + case value.Type().ConvertibleTo(elem.Type()): + elem.Set(value.Convert(elem.Type())) + default: + return fmt.Errorf("cannot assign %s to %s", value.Type(), elem.Type()) + } + } + + return nil +} + +func transferScanValues(transfer models.Transfer) []any { + return []any{ + transfer.ID, + transfer.Status, + transfer.WrappedRootKey, + transfer.ManifestObjectKey, + transfer.ManifestCiphertextSize, + transfer.TotalFiles, + transfer.TotalCiphertextBytes, + transfer.ExpiresAt, + transfer.CreatedAt, + transfer.UpdatedAt, + transfer.FinalizedAt, + transfer.ManageTokenHash, + transfer.DeletedAt, + transfer.PurgedAt, + } +} + +func fileScanValues(file models.TransferFile) []any { + return []any{ + file.TransferID, + file.OpaqueFileID, + file.TotalChunks, + file.CiphertextBytes, + file.PlaintextBytes, + file.ChunkSize, + file.UploadStatus, + file.CreatedAt, + file.UpdatedAt, + } +} diff --git a/apps/api/internal/service/transfer_service.go b/apps/api/internal/service/transfer_service.go new file mode 100644 index 0000000..ac2a1e9 --- /dev/null +++ b/apps/api/internal/service/transfer_service.go @@ -0,0 +1,716 @@ +package service + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "github.com/xdrop/monorepo/internal/config" + "github.com/xdrop/monorepo/internal/models" + "github.com/xdrop/monorepo/internal/ratelimit" + "github.com/xdrop/monorepo/internal/repo" + "github.com/xdrop/monorepo/internal/storage" +) + +var readRandom = rand.Read + +// Service coordinates transfer lifecycle rules across storage, persistence, and rate limits. +type Service struct { + cfg config.Config + repo repo.Repository + storage storage.ObjectStorage + limiter ratelimit.Limiter +} + +// New builds the application service with its persistence, storage, and rate-limit dependencies. +func New(cfg config.Config, repository repo.Repository, objectStorage storage.ObjectStorage, limiter ratelimit.Limiter) *Service { + return &Service{ + cfg: cfg, + repo: repository, + storage: objectStorage, + limiter: limiter, + } +} + +// UploadConfig advertises browser upload constraints derived from server policy. +type UploadConfig struct { + ChunkSize int64 `json:"chunkSize"` + MaxParallel int `json:"maxParallel"` + MaxFileCount int `json:"maxFileCount"` + MaxTransferBytes int64 `json:"maxTransferBytes"` +} + +// CreateTransferRequest starts a new transfer with a supported expiry option. +type CreateTransferRequest struct { + ExpiresInSeconds int `json:"expiresInSeconds"` + ExpiresInDays int `json:"expiresInDays,omitempty"` +} + +// CreateTransferResponse returns the transfer handle, manage token, and upload limits. +type CreateTransferResponse struct { + TransferID string `json:"transferId"` + ManageToken string `json:"manageToken"` + UploadConfig UploadConfig `json:"uploadConfig"` + ExpiresAt time.Time `json:"expiresAt"` +} + +// RegisterFileRequest describes one encrypted file before chunk upload begins. +type RegisterFileRequest struct { + FileID string `json:"fileId"` + TotalChunks int `json:"totalChunks"` + CiphertextBytes int64 `json:"ciphertextBytes"` + PlaintextBytes *int64 `json:"plaintextBytes"` + ChunkSize int64 `json:"chunkSize"` +} + +// UploadURLRequest asks the server to presign upload URLs for a batch of chunks. +type UploadURLRequest struct { + Chunks []UploadChunkRequest `json:"chunks"` +} + +// UploadChunkRequest identifies one chunk within a file. +type UploadChunkRequest struct { + FileID string `json:"fileId"` + ChunkIndex int `json:"chunkIndex"` +} + +// UploadURLItem contains the storage coordinates for an uploadable chunk. +type UploadURLItem struct { + FileID string `json:"fileId"` + ChunkIndex int `json:"chunkIndex"` + ObjectKey string `json:"objectKey"` + URL string `json:"url"` +} + +// CompleteChunkRequest records a successfully uploaded chunk and its checksum. +type CompleteChunkRequest struct { + FileID string `json:"fileId"` + ChunkIndex int `json:"chunkIndex"` + CiphertextSize int64 `json:"ciphertextSize"` + ChecksumSHA256 string `json:"checksumSha256"` +} + +// ManifestUploadRequest uploads the encrypted manifest envelope as base64 text. +type ManifestUploadRequest struct { + CiphertextBase64 string `json:"ciphertextBase64"` +} + +// FinalizeTransferRequest promotes a complete upload into a downloadable transfer. +type FinalizeTransferRequest struct { + WrappedRootKey string `json:"wrappedRootKey"` + TotalFiles int `json:"totalFiles"` + TotalCiphertextBytes int64 `json:"totalCiphertextBytes"` +} + +// UpdateTransferRequest describes the mutable parts of a transfer after creation. +type UpdateTransferRequest struct { + ExpiresInSeconds *int `json:"expiresInSeconds"` + ExpiresInDays *int `json:"expiresInDays,omitempty"` + CiphertextBase64 string `json:"ciphertextBase64"` +} + +// ManageTransferResponse exposes resume and management state to the transfer owner. +type ManageTransferResponse struct { + ID string `json:"id"` + Status models.TransferStatus `json:"status"` + ExpiresAt time.Time `json:"expiresAt"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + FinalizedAt *time.Time `json:"finalizedAt,omitempty"` + ManifestCiphertextSize int64 `json:"manifestCiphertextSize"` + TotalFiles int `json:"totalFiles"` + TotalCiphertextBytes int64 `json:"totalCiphertextBytes"` + Files []ManageTransferFile `json:"files"` + UploadedChunks map[string][]int `json:"uploadedChunks,omitempty"` +} + +// ManageTransferFile reports per-file upload progress for the owner view. +type ManageTransferFile struct { + FileID string `json:"fileId"` + TotalChunks int `json:"totalChunks"` + CiphertextBytes int64 `json:"ciphertextBytes"` + ChunkSize int64 `json:"chunkSize"` + UploadStatus string `json:"uploadStatus"` +} + +// PublicTransferResponse exposes the public download state without revealing manage controls. +type PublicTransferResponse struct { + ID string `json:"id"` + Status string `json:"status"` + ExpiresAt time.Time `json:"expiresAt"` + WrappedRootKey string `json:"wrappedRootKey,omitempty"` + ManifestURL string `json:"manifestUrl,omitempty"` + ManifestCiphertextSize int64 `json:"manifestCiphertextSize,omitempty"` + DownloadConfig PublicDownloadConfig `json:"downloadConfig,omitempty"` +} + +// PublicDownloadConfig advertises public download behavior that the client should honor. +type PublicDownloadConfig struct { + PresignTTLSeconds int `json:"presignTtlSeconds"` +} + +// DownloadURLRequest asks for download URLs for specific uploaded chunks. +type DownloadURLRequest struct { + Chunks []UploadChunkRequest `json:"chunks"` +} + +// DownloadURLItem returns a public download URL for one chunk. +type DownloadURLItem struct { + FileID string `json:"fileId"` + ChunkIndex int `json:"chunkIndex"` + URL string `json:"url"` +} + +// HTTPError carries an HTTP status and machine-readable API code through the service layer. +type HTTPError struct { + Status int + Code string + Message string +} + +func (e *HTTPError) Error() string { + return e.Message +} + +// CreateTransfer creates a draft transfer and returns its manage token and upload policy. +func (s *Service) CreateTransfer(ctx context.Context, clientKey string, request CreateTransferRequest) (CreateTransferResponse, error) { + if err := s.enforceRateLimit(ctx, "create:"+clientKey, s.cfg.CreateLimit, time.Hour); err != nil { + return CreateTransferResponse{}, err + } + + expiryDuration, err := requestedCreateExpiry(request, s.cfg.DefaultExpiry) + if err != nil { + return CreateTransferResponse{}, err + } + + transferID, err := randomToken(18) + if err != nil { + return CreateTransferResponse{}, fmt.Errorf("generate transfer id: %w", err) + } + manageToken, err := randomToken(32) + if err != nil { + return CreateTransferResponse{}, fmt.Errorf("generate manage token: %w", err) + } + + now := time.Now().UTC() + expiresAt := now.Add(expiryDuration) + transfer := models.Transfer{ + ID: transferID, + Status: models.TransferStatusDraft, + ExpiresAt: expiresAt, + CreatedAt: now, + UpdatedAt: now, + ManageTokenHash: hashToken(manageToken), + } + + if err := s.repo.CreateTransfer(ctx, transfer); err != nil { + return CreateTransferResponse{}, fmt.Errorf("create transfer: %w", err) + } + + return CreateTransferResponse{ + TransferID: transferID, + ManageToken: manageToken, + UploadConfig: UploadConfig{ + ChunkSize: s.cfg.ChunkSize, + MaxParallel: 6, + MaxFileCount: s.cfg.MaxFileCount, + MaxTransferBytes: s.cfg.MaxTransferBytes, + }, + ExpiresAt: expiresAt, + }, nil +} + +// RegisterFiles validates encrypted file metadata before upload URLs are issued. +func (s *Service) RegisterFiles(ctx context.Context, transferID string, token string, files []RegisterFileRequest) error { + if len(files) == 0 { + return &HTTPError{Status: 400, Code: "empty_files", Message: "at least one file must be registered"} + } + if len(files) > s.cfg.MaxFileCount { + return &HTTPError{Status: 400, Code: "too_many_files", Message: "file count exceeds configured maximum"} + } + + transfer, err := s.authorizeManage(ctx, transferID, token) + if err != nil { + return err + } + if isExpired(transfer) { + return &HTTPError{Status: 410, Code: "expired", Message: "transfer has expired"} + } + + totalBytes := int64(0) + modelFiles := make([]models.TransferFile, 0, len(files)) + for _, file := range files { + if strings.TrimSpace(file.FileID) == "" || file.TotalChunks <= 0 || file.ChunkSize <= 0 || file.CiphertextBytes <= 0 { + return &HTTPError{Status: 400, Code: "invalid_file_registration", Message: "file registration contains invalid values"} + } + totalBytes += file.CiphertextBytes + modelFiles = append(modelFiles, models.TransferFile{ + TransferID: transferID, + OpaqueFileID: file.FileID, + TotalChunks: file.TotalChunks, + CiphertextBytes: file.CiphertextBytes, + PlaintextBytes: file.PlaintextBytes, + ChunkSize: file.ChunkSize, + UploadStatus: "pending", + }) + } + + if totalBytes > s.cfg.MaxTransferBytes { + return &HTTPError{Status: 400, Code: "transfer_too_large", Message: "transfer exceeds configured size limit"} + } + + if err := s.repo.RegisterFiles(ctx, transferID, modelFiles); err != nil { + return fmt.Errorf("register files: %w", err) + } + + return nil +} + +// CreateUploadURLs presigns upload destinations for a managed transfer. +func (s *Service) CreateUploadURLs(ctx context.Context, transferID string, token string, request UploadURLRequest) ([]UploadURLItem, error) { + transfer, err := s.authorizeManage(ctx, transferID, token) + if err != nil { + return nil, err + } + if isExpired(transfer) { + return nil, &HTTPError{Status: 410, Code: "expired", Message: "transfer has expired"} + } + + files, err := s.repo.ListFiles(ctx, transferID) + if err != nil { + return nil, fmt.Errorf("list files: %w", err) + } + fileIndex := map[string]models.TransferFile{} + for _, file := range files { + fileIndex[file.OpaqueFileID] = file + } + + urls := make([]UploadURLItem, 0, len(request.Chunks)) + for _, chunk := range request.Chunks { + file, ok := fileIndex[chunk.FileID] + if !ok || chunk.ChunkIndex < 0 || chunk.ChunkIndex >= file.TotalChunks { + return nil, &HTTPError{Status: 400, Code: "invalid_chunk_request", Message: "upload chunk request is invalid"} + } + + objectKey := chunkObjectKey(transferID, chunk.FileID, chunk.ChunkIndex) + url, err := s.storage.PresignUpload(ctx, objectKey, s.cfg.PresignTTL) + if err != nil { + return nil, fmt.Errorf("presign upload %s/%d: %w", chunk.FileID, chunk.ChunkIndex, err) + } + + urls = append(urls, UploadURLItem{ + FileID: chunk.FileID, + ChunkIndex: chunk.ChunkIndex, + ObjectKey: objectKey, + URL: url, + }) + } + + return urls, nil +} + +// CompleteChunks records uploaded chunk metadata for resume and integrity checks. +func (s *Service) CompleteChunks(ctx context.Context, transferID string, token string, chunks []CompleteChunkRequest) error { + transfer, err := s.authorizeManage(ctx, transferID, token) + if err != nil { + return err + } + if isExpired(transfer) { + return &HTTPError{Status: 410, Code: "expired", Message: "transfer has expired"} + } + + modelChunks := make([]models.TransferChunk, 0, len(chunks)) + for _, chunk := range chunks { + if strings.TrimSpace(chunk.FileID) == "" || chunk.ChunkIndex < 0 || chunk.CiphertextSize <= 0 || strings.TrimSpace(chunk.ChecksumSHA256) == "" { + return &HTTPError{Status: 400, Code: "invalid_chunk_completion", Message: "chunk completion payload is invalid"} + } + modelChunks = append(modelChunks, models.TransferChunk{ + TransferID: transferID, + OpaqueFileID: chunk.FileID, + ChunkIndex: chunk.ChunkIndex, + ObjectKey: chunkObjectKey(transferID, chunk.FileID, chunk.ChunkIndex), + CiphertextSize: chunk.CiphertextSize, + ChecksumSHA256: chunk.ChecksumSHA256, + }) + } + + if err := s.repo.CompleteChunks(ctx, transferID, modelChunks); err != nil { + return fmt.Errorf("complete chunks: %w", err) + } + + return nil +} + +// PutManifest stores the encrypted manifest and records its object location. +func (s *Service) PutManifest(ctx context.Context, transferID string, token string, request ManifestUploadRequest) error { + transfer, err := s.authorizeManage(ctx, transferID, token) + if err != nil { + return err + } + if isExpired(transfer) { + return &HTTPError{Status: 410, Code: "expired", Message: "transfer has expired"} + } + + payload, err := base64.StdEncoding.DecodeString(request.CiphertextBase64) + if err != nil || len(payload) == 0 { + return &HTTPError{Status: 400, Code: "invalid_manifest", Message: "manifest payload must be valid base64 ciphertext"} + } + + objectKey := manifestObjectKey(transferID) + if err := s.storage.PutObject(ctx, objectKey, payload, "application/octet-stream"); err != nil { + return fmt.Errorf("put manifest: %w", err) + } + if err := s.repo.SetManifest(ctx, transferID, objectKey, int64(len(payload))); err != nil { + return fmt.Errorf("set manifest: %w", err) + } + + return nil +} + +// FinalizeTransfer marks a fully uploaded transfer as ready for public downloads. +func (s *Service) FinalizeTransfer(ctx context.Context, transferID string, token string, request FinalizeTransferRequest) error { + transfer, err := s.authorizeManage(ctx, transferID, token) + if err != nil { + return err + } + if isExpired(transfer) { + return &HTTPError{Status: 410, Code: "expired", Message: "transfer has expired"} + } + if strings.TrimSpace(request.WrappedRootKey) == "" || request.TotalFiles <= 0 || request.TotalCiphertextBytes <= 0 { + return &HTTPError{Status: 400, Code: "invalid_finalize", Message: "finalize payload is invalid"} + } + + if err := s.repo.FinalizeTransfer(ctx, transferID, request.WrappedRootKey, request.TotalFiles, request.TotalCiphertextBytes); err != nil { + return err + } + + return nil +} + +// GetManageTransfer returns owner-facing transfer state, including uploaded chunk indexes. +func (s *Service) GetManageTransfer(ctx context.Context, transferID string, token string) (ManageTransferResponse, error) { + if _, err := s.authorizeManage(ctx, transferID, token); err != nil { + return ManageTransferResponse{}, err + } + + resume, err := s.repo.GetResumeState(ctx, transferID) + if err != nil { + return ManageTransferResponse{}, err + } + + files := make([]ManageTransferFile, 0, len(resume.Files)) + for _, file := range resume.Files { + files = append(files, ManageTransferFile{ + FileID: file.OpaqueFileID, + TotalChunks: file.TotalChunks, + CiphertextBytes: file.CiphertextBytes, + ChunkSize: file.ChunkSize, + UploadStatus: file.UploadStatus, + }) + } + + return ManageTransferResponse{ + ID: resume.Transfer.ID, + Status: publicStatus(resume.Transfer), + ExpiresAt: resume.Transfer.ExpiresAt, + CreatedAt: resume.Transfer.CreatedAt, + UpdatedAt: resume.Transfer.UpdatedAt, + FinalizedAt: resume.Transfer.FinalizedAt, + ManifestCiphertextSize: resume.Transfer.ManifestCiphertextSize, + TotalFiles: resume.Transfer.TotalFiles, + TotalCiphertextBytes: resume.Transfer.TotalCiphertextBytes, + Files: files, + UploadedChunks: resume.UploadedChunks, + }, nil +} + +// ResumeTransfer is an alias for GetManageTransfer used by the browser resume flow. +func (s *Service) ResumeTransfer(ctx context.Context, transferID string, token string) (ManageTransferResponse, error) { + return s.GetManageTransfer(ctx, transferID, token) +} + +// UpdateTransfer changes supported mutable fields such as expiry or manifest ciphertext. +func (s *Service) UpdateTransfer(ctx context.Context, transferID string, token string, request UpdateTransferRequest) error { + transfer, err := s.authorizeManage(ctx, transferID, token) + if err != nil { + return err + } + if publicStatus(transfer) == models.TransferStatusDeleted { + return &HTTPError{Status: 410, Code: "deleted", Message: "transfer has been deleted"} + } + + params := models.UpdateTransferParams{} + if expiryDuration, shouldUpdateExpiry, err := requestedUpdateExpiry(request); err != nil { + return err + } else if shouldUpdateExpiry { + expiresAt := time.Now().UTC().Add(expiryDuration) + params.ExpiresAt = &expiresAt + } + + if strings.TrimSpace(request.CiphertextBase64) != "" { + payload, err := base64.StdEncoding.DecodeString(request.CiphertextBase64) + if err != nil || len(payload) == 0 { + return &HTTPError{Status: 400, Code: "invalid_manifest", Message: "updated manifest must be valid base64 ciphertext"} + } + objectKey := transfer.ManifestObjectKey + if objectKey == "" { + objectKey = manifestObjectKey(transferID) + } + if err := s.storage.PutObject(ctx, objectKey, payload, "application/octet-stream"); err != nil { + return fmt.Errorf("put updated manifest: %w", err) + } + params.ManifestObjectKey = &objectKey + size := int64(len(payload)) + params.ManifestCiphertextSize = &size + } + + if err := s.repo.UpdateTransfer(ctx, transferID, params); err != nil { + return fmt.Errorf("update transfer: %w", err) + } + + return nil +} + +// DeleteTransfer tombstones a transfer and best-effort removes its remote objects. +func (s *Service) DeleteTransfer(ctx context.Context, transferID string, token string) error { + if _, err := s.authorizeManage(ctx, transferID, token); err != nil { + return err + } + + if err := s.repo.MarkDeleted(ctx, transferID); err != nil { + return fmt.Errorf("delete transfer: %w", err) + } + _ = s.storage.DeletePrefix(ctx, transferPrefix(transferID)) + + return nil +} + +// GetPublicTransfer returns the public download descriptor for recipients. +func (s *Service) GetPublicTransfer(ctx context.Context, clientKey string, transferID string) (PublicTransferResponse, error) { + if err := s.enforceRateLimit(ctx, "public:"+clientKey+":"+transferID, s.cfg.PublicReadLimit, time.Minute); err != nil { + return PublicTransferResponse{}, err + } + + transfer, err := s.repo.GetTransfer(ctx, transferID) + if errors.Is(err, repo.ErrNotFound) { + return PublicTransferResponse{}, &HTTPError{Status: 404, Code: "not_found", Message: "transfer not found"} + } + if err != nil { + return PublicTransferResponse{}, err + } + + status := publicStatus(transfer) + response := PublicTransferResponse{ + ID: transfer.ID, + Status: string(status), + ExpiresAt: transfer.ExpiresAt, + } + + if status != models.TransferStatusReady { + return response, nil + } + + manifestURL, err := s.storage.PresignDownload(ctx, transfer.ManifestObjectKey, s.cfg.PresignTTL) + if err != nil { + return PublicTransferResponse{}, fmt.Errorf("presign manifest download: %w", err) + } + + response.WrappedRootKey = transfer.WrappedRootKey + response.ManifestURL = manifestURL + response.ManifestCiphertextSize = transfer.ManifestCiphertextSize + response.DownloadConfig = PublicDownloadConfig{PresignTTLSeconds: int(s.cfg.PresignTTL.Seconds())} + + return response, nil +} + +// CreateDownloadURLs presigns recipient download URLs for chunks that actually exist. +func (s *Service) CreateDownloadURLs(ctx context.Context, clientKey string, transferID string, request DownloadURLRequest) ([]DownloadURLItem, error) { + if err := s.enforceRateLimit(ctx, "download:"+clientKey+":"+transferID, s.cfg.DownloadURLLimit, time.Minute); err != nil { + return nil, err + } + + resume, err := s.repo.GetResumeState(ctx, transferID) + if errors.Is(err, repo.ErrNotFound) { + return nil, &HTTPError{Status: 404, Code: "not_found", Message: "transfer not found"} + } + if err != nil { + return nil, err + } + if publicStatus(resume.Transfer) != models.TransferStatusReady { + return nil, &HTTPError{Status: 409, Code: "transfer_unavailable", Message: "transfer is not available for download"} + } + + uploaded := map[string]map[int]struct{}{} + for fileID, chunks := range resume.UploadedChunks { + uploaded[fileID] = map[int]struct{}{} + for _, chunkIndex := range chunks { + uploaded[fileID][chunkIndex] = struct{}{} + } + } + + urls := make([]DownloadURLItem, 0, len(request.Chunks)) + for _, chunk := range request.Chunks { + chunkSet, ok := uploaded[chunk.FileID] + if !ok { + return nil, &HTTPError{Status: 400, Code: "invalid_download_request", Message: "requested file is unavailable"} + } + if _, ok = chunkSet[chunk.ChunkIndex]; !ok { + return nil, &HTTPError{Status: 400, Code: "invalid_download_request", Message: "requested chunk is unavailable"} + } + + url, err := s.storage.PresignDownload(ctx, chunkObjectKey(transferID, chunk.FileID, chunk.ChunkIndex), s.cfg.PresignTTL) + if err != nil { + return nil, fmt.Errorf("presign chunk download: %w", err) + } + urls = append(urls, DownloadURLItem{ + FileID: chunk.FileID, + ChunkIndex: chunk.ChunkIndex, + URL: url, + }) + } + + return urls, nil +} + +// CleanupExpired removes remote objects for expired or deleted transfers and marks them purged. +func (s *Service) CleanupExpired(ctx context.Context) error { + transfers, err := s.repo.ListCleanupCandidates(ctx, 200) + if err != nil { + return fmt.Errorf("list cleanup candidates: %w", err) + } + + for _, transfer := range transfers { + if err := s.storage.DeletePrefix(ctx, transferPrefix(transfer.ID)); err != nil { + return fmt.Errorf("delete transfer objects %s: %w", transfer.ID, err) + } + if err := s.repo.MarkPurged(ctx, transfer.ID); err != nil { + return fmt.Errorf("mark purged %s: %w", transfer.ID, err) + } + } + + return nil +} + +// authorizeManage validates the owner token using a constant-time hash comparison. +func (s *Service) authorizeManage(ctx context.Context, transferID string, token string) (models.Transfer, error) { + if strings.TrimSpace(token) == "" { + return models.Transfer{}, &HTTPError{Status: 401, Code: "missing_manage_token", Message: "manage token is required"} + } + + transfer, err := s.repo.GetTransfer(ctx, transferID) + if errors.Is(err, repo.ErrNotFound) { + return models.Transfer{}, &HTTPError{Status: 404, Code: "not_found", Message: "transfer not found"} + } + if err != nil { + return models.Transfer{}, err + } + + givenHash := hashToken(token) + if subtle.ConstantTimeCompare([]byte(givenHash), []byte(transfer.ManageTokenHash)) != 1 { + return models.Transfer{}, &HTTPError{Status: 403, Code: "invalid_manage_token", Message: "manage token is invalid"} + } + + return transfer, nil +} + +// enforceRateLimit translates limiter decisions into API errors. +func (s *Service) enforceRateLimit(ctx context.Context, key string, limit int, window time.Duration) error { + allowed, err := s.limiter.Allow(ctx, key, limit, window) + if err != nil { + return fmt.Errorf("rate limit check: %w", err) + } + if !allowed { + return &HTTPError{Status: 429, Code: "rate_limited", Message: "too many requests"} + } + + return nil +} + +// hashToken stores manage tokens as SHA-256 digests instead of plaintext. +func hashToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +// randomToken generates URL-safe opaque identifiers for transfers and manage tokens. +func randomToken(size int) (string, error) { + buffer := make([]byte, size) + if _, err := readRandom(buffer); err != nil { + return "", err + } + + return base64.RawURLEncoding.EncodeToString(buffer), nil +} + +// publicStatus collapses internal states into the limited public status model. +func publicStatus(transfer models.Transfer) models.TransferStatus { + if transfer.DeletedAt != nil || transfer.Status == models.TransferStatusDeleted { + return models.TransferStatusDeleted + } + if isExpired(transfer) { + return models.TransferStatusExpired + } + if transfer.Status != models.TransferStatusReady || transfer.ManifestObjectKey == "" || transfer.WrappedRootKey == "" { + return models.TransferStatusIncomplete + } + + return models.TransferStatusReady +} + +func isExpired(transfer models.Transfer) bool { + return transfer.ExpiresAt.Before(time.Now().UTC()) +} + +func requestedCreateExpiry(request CreateTransferRequest, fallback time.Duration) (time.Duration, error) { + if request.ExpiresInSeconds > 0 { + return validateExpiryDuration(time.Duration(request.ExpiresInSeconds) * time.Second) + } + if request.ExpiresInDays > 0 { + return validateExpiryDuration(time.Duration(request.ExpiresInDays) * 24 * time.Hour) + } + + return validateExpiryDuration(fallback) +} + +func requestedUpdateExpiry(request UpdateTransferRequest) (time.Duration, bool, error) { + if request.ExpiresInSeconds != nil { + duration, err := validateExpiryDuration(time.Duration(*request.ExpiresInSeconds) * time.Second) + return duration, true, err + } + if request.ExpiresInDays != nil { + duration, err := validateExpiryDuration(time.Duration(*request.ExpiresInDays) * 24 * time.Hour) + return duration, true, err + } + + return 0, false, nil +} + +func validateExpiryDuration(duration time.Duration) (time.Duration, error) { + if !config.IsAllowedExpiry(duration) { + return 0, &HTTPError{Status: 400, Code: "invalid_expiry", Message: "expiry must match a supported option"} + } + + return duration, nil +} + +// manifestObjectKey is the canonical object path for an encrypted manifest. +func manifestObjectKey(transferID string) string { + return fmt.Sprintf("transfers/%s/manifest.bin", transferID) +} + +// chunkObjectKey is the canonical object path for one encrypted chunk. +func chunkObjectKey(transferID string, fileID string, chunkIndex int) string { + return fmt.Sprintf("transfers/%s/files/%s/chunks/%08d.bin", transferID, fileID, chunkIndex) +} + +// transferPrefix returns the storage prefix containing every object for a transfer. +func transferPrefix(transferID string) string { + return fmt.Sprintf("transfers/%s/", transferID) +} diff --git a/apps/api/internal/service/transfer_service_additional_test.go b/apps/api/internal/service/transfer_service_additional_test.go new file mode 100644 index 0000000..28b183b --- /dev/null +++ b/apps/api/internal/service/transfer_service_additional_test.go @@ -0,0 +1,916 @@ +package service + +import ( + "context" + "encoding/base64" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/models" + "github.com/xdrop/monorepo/internal/ratelimit" + "github.com/xdrop/monorepo/internal/repo" +) + +func TestHTTPErrorMessageAndExpiryHelpers(t *testing.T) { + t.Parallel() + + t.Run("http error exposes message", func(t *testing.T) { + t.Parallel() + + err := &HTTPError{Message: "plain message"} + require.Equal(t, "plain message", err.Error()) + }) + + t.Run("requested create expiry supports days and fallback", func(t *testing.T) { + t.Parallel() + + duration, err := requestedCreateExpiry(CreateTransferRequest{ExpiresInDays: 1}, 12*time.Hour) + require.NoError(t, err) + require.Equal(t, 24*time.Hour, duration) + + duration, err = requestedCreateExpiry(CreateTransferRequest{}, 7*24*time.Hour) + require.NoError(t, err) + require.Equal(t, 7*24*time.Hour, duration) + }) + + t.Run("requested update expiry supports days and noop", func(t *testing.T) { + t.Parallel() + + days := 7 + duration, shouldUpdate, err := requestedUpdateExpiry(UpdateTransferRequest{ExpiresInDays: &days}) + require.NoError(t, err) + require.True(t, shouldUpdate) + require.Equal(t, 7*24*time.Hour, duration) + + duration, shouldUpdate, err = requestedUpdateExpiry(UpdateTransferRequest{}) + require.NoError(t, err) + require.False(t, shouldUpdate) + require.Zero(t, duration) + }) +} + +func TestCreateTransferPropagatesLimiterAndRepositoryErrors(t *testing.T) { + t.Parallel() + + t.Run("limiter error", func(t *testing.T) { + t.Parallel() + + svc := New(testConfig(), newMemoryRepository(), &memoryStorage{}, errorLimiter{err: errors.New("redis unavailable")}) + + _, err := svc.CreateTransfer(context.Background(), "198.51.100.10", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.ErrorContains(t, err, "rate limit check") + }) + + t.Run("repository error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + createTransferErr: errors.New("insert failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + + _, err := svc.CreateTransfer(context.Background(), "198.51.100.10", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.ErrorContains(t, err, "create transfer") + }) +} + +func TestCreateTransferPropagatesRandomTokenFailures(t *testing.T) { + originalReadRandom := readRandom + t.Cleanup(func() { + readRandom = originalReadRandom + }) + + t.Run("transfer id generation", func(t *testing.T) { + readRandom = func([]byte) (int, error) { + return 0, errors.New("entropy failed") + } + + svc := newTestService(newMemoryRepository()) + _, err := svc.CreateTransfer(context.Background(), "198.51.100.10", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.ErrorContains(t, err, "generate transfer id") + }) + + t.Run("manage token generation", func(t *testing.T) { + calls := 0 + readRandom = func(buffer []byte) (int, error) { + calls++ + if calls == 2 { + return 0, errors.New("entropy failed") + } + for index := range buffer { + buffer[index] = byte(index + 1) + } + return len(buffer), nil + } + + svc := newTestService(newMemoryRepository()) + _, err := svc.CreateTransfer(context.Background(), "198.51.100.10", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.ErrorContains(t, err, "generate manage token") + }) +} + +func TestCreateUploadURLsHandlesExpiryAndStorageFailures(t *testing.T) { + t.Parallel() + + t.Run("expired transfer", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + expireTransfer(t, repository, create.TransferID) + + _, err := svc.CreateUploadURLs(context.Background(), create.TransferID, create.ManageToken, UploadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + requireHTTPError(t, err, 410, "expired") + }) + + t.Run("presign failure", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &failingStorage{ + memoryStorage: &memoryStorage{}, + presignUploadErr: errors.New("presign upload failed"), + } + svc := New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + + _, err := svc.CreateUploadURLs(context.Background(), create.TransferID, create.ManageToken, UploadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + require.ErrorContains(t, err, "presign upload") + }) + + t.Run("list files failure", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + listFilesErr: errors.New("query failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + + _, err := svc.CreateUploadURLs(context.Background(), create.TransferID, create.ManageToken, UploadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + require.ErrorContains(t, err, "list files") + }) +} + +func TestCreateUploadURLsReturnsPresignedItems(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + + items, err := svc.CreateUploadURLs(context.Background(), create.TransferID, create.ManageToken, UploadURLRequest{ + Chunks: []UploadChunkRequest{ + {FileID: "file-a", ChunkIndex: 0}, + {FileID: "file-a", ChunkIndex: 1}, + }, + }) + require.NoError(t, err) + require.Len(t, items, 2) + require.Equal(t, "file-a", items[0].FileID) + require.Equal(t, 0, items[0].ChunkIndex) + require.Contains(t, items[0].URL, chunkObjectKey(create.TransferID, "file-a", 0)) +} + +func TestCompleteChunksHandlesExpiryAndRepositoryFailures(t *testing.T) { + t.Parallel() + + t.Run("expired transfer", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + expireTransfer(t, repository, create.TransferID) + + err := svc.CompleteChunks(context.Background(), create.TransferID, create.ManageToken, []CompleteChunkRequest{{ + FileID: "file-a", + ChunkIndex: 0, + CiphertextSize: 32, + ChecksumSHA256: "checksum", + }}) + requireHTTPError(t, err, 410, "expired") + }) + + t.Run("repository error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + completeChunksErr: errors.New("upsert failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + + err := svc.CompleteChunks(context.Background(), create.TransferID, create.ManageToken, []CompleteChunkRequest{{ + FileID: "file-a", + ChunkIndex: 0, + CiphertextSize: 32, + ChecksumSHA256: "checksum", + }}) + require.ErrorContains(t, err, "complete chunks") + }) + + t.Run("repository lookup error during authorization", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{memoryRepository: newMemoryRepository()} + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + repository.getTransferErr = errors.New("lookup failed") + + err := svc.CompleteChunks(context.Background(), create.TransferID, create.ManageToken, []CompleteChunkRequest{{ + FileID: "file-a", + ChunkIndex: 0, + CiphertextSize: 32, + ChecksumSHA256: "checksum", + }}) + require.ErrorContains(t, err, "lookup failed") + }) +} + +func TestManageAuthorizedMethodsPropagateAuthorizationFailures(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + requireHTTPError( + t, + svc.RegisterFiles(context.Background(), create.TransferID, "", []RegisterFileRequest{{ + FileID: "file-a", + TotalChunks: 1, + CiphertextBytes: 32, + ChunkSize: 32, + }}), + 401, + "missing_manage_token", + ) + + _, err := svc.CreateUploadURLs(context.Background(), create.TransferID, "", UploadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + requireHTTPError(t, err, 401, "missing_manage_token") + + err = svc.PutManifest(context.Background(), create.TransferID, "", ManifestUploadRequest{ + CiphertextBase64: base64.StdEncoding.EncodeToString([]byte("manifest")), + }) + requireHTTPError(t, err, 401, "missing_manage_token") + + err = svc.FinalizeTransfer(context.Background(), create.TransferID, "", FinalizeTransferRequest{ + WrappedRootKey: "wrapped", + TotalFiles: 1, + TotalCiphertextBytes: 32, + }) + requireHTTPError(t, err, 401, "missing_manage_token") + + err = svc.UpdateTransfer(context.Background(), create.TransferID, "", UpdateTransferRequest{}) + requireHTTPError(t, err, 401, "missing_manage_token") +} + +func TestCompleteChunksRejectsInvalidPayload(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + err := svc.CompleteChunks(context.Background(), create.TransferID, create.ManageToken, []CompleteChunkRequest{{ + FileID: " ", + ChunkIndex: 0, + CiphertextSize: 32, + ChecksumSHA256: "checksum", + }}) + requireHTTPError(t, err, 400, "invalid_chunk_completion") +} + +func TestPutManifestHandlesExpiryAndFailures(t *testing.T) { + t.Parallel() + + t.Run("expired transfer", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + expireTransfer(t, repository, create.TransferID) + + err := svc.PutManifest(context.Background(), create.TransferID, create.ManageToken, ManifestUploadRequest{ + CiphertextBase64: base64.StdEncoding.EncodeToString([]byte("manifest")), + }) + requireHTTPError(t, err, 410, "expired") + }) + + t.Run("storage error", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &failingStorage{ + memoryStorage: &memoryStorage{}, + putObjectErr: errors.New("upload failed"), + } + svc := New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + + err := svc.PutManifest(context.Background(), create.TransferID, create.ManageToken, ManifestUploadRequest{ + CiphertextBase64: base64.StdEncoding.EncodeToString([]byte("manifest")), + }) + require.ErrorContains(t, err, "put manifest") + }) + + t.Run("repository error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + setManifestErr: errors.New("update failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + + err := svc.PutManifest(context.Background(), create.TransferID, create.ManageToken, ManifestUploadRequest{ + CiphertextBase64: base64.StdEncoding.EncodeToString([]byte("manifest")), + }) + require.ErrorContains(t, err, "set manifest") + }) +} + +func TestFinalizeTransferHandlesExpiryAndRepositoryFailures(t *testing.T) { + t.Parallel() + + t.Run("expired transfer", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + expireTransfer(t, repository, create.TransferID) + + err := svc.FinalizeTransfer(context.Background(), create.TransferID, create.ManageToken, FinalizeTransferRequest{ + WrappedRootKey: "wrapped", + TotalFiles: 1, + TotalCiphertextBytes: 32, + }) + requireHTTPError(t, err, 410, "expired") + }) + + t.Run("repository error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + } + storage := &memoryStorage{} + svc := New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + require.NoError(t, svc.CompleteChunks(context.Background(), create.TransferID, create.ManageToken, []CompleteChunkRequest{ + {FileID: "file-a", ChunkIndex: 0, CiphertextSize: 32, ChecksumSHA256: "a"}, + {FileID: "file-a", ChunkIndex: 1, CiphertextSize: 32, ChecksumSHA256: "b"}, + })) + require.NoError(t, svc.PutManifest(context.Background(), create.TransferID, create.ManageToken, ManifestUploadRequest{ + CiphertextBase64: base64.StdEncoding.EncodeToString([]byte("manifest")), + })) + repository.finalizeErr = errors.New("finalize failed") + + err := svc.FinalizeTransfer(context.Background(), create.TransferID, create.ManageToken, FinalizeTransferRequest{ + WrappedRootKey: "wrapped", + TotalFiles: 1, + TotalCiphertextBytes: 64, + }) + require.ErrorContains(t, err, "finalize failed") + }) +} + +func TestGetManageTransferAndResumeTransferReturnState(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + require.NoError(t, svc.CompleteChunks(context.Background(), create.TransferID, create.ManageToken, []CompleteChunkRequest{{ + FileID: "file-a", + ChunkIndex: 0, + CiphertextSize: 32, + ChecksumSHA256: "checksum", + }})) + + manage, err := svc.GetManageTransfer(context.Background(), create.TransferID, create.ManageToken) + require.NoError(t, err) + require.Equal(t, create.TransferID, manage.ID) + require.Len(t, manage.Files, 1) + require.Equal(t, "file-a", manage.Files[0].FileID) + require.Equal(t, []int{0}, manage.UploadedChunks["file-a"]) + + resume, err := svc.ResumeTransfer(context.Background(), create.TransferID, create.ManageToken) + require.NoError(t, err) + require.Equal(t, manage, resume) +} + +func TestGetManageTransferPropagatesResumeErrors(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + getResumeStateErr: errors.New("resume failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + + _, err := svc.GetManageTransfer(context.Background(), create.TransferID, create.ManageToken) + require.ErrorContains(t, err, "resume failed") +} + +func TestUpdateTransferHandlesInvalidManifestAndRepositoryErrors(t *testing.T) { + t.Parallel() + + t.Run("invalid manifest", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + expirySeconds := 3600 + + err := svc.UpdateTransfer(context.Background(), create.TransferID, create.ManageToken, UpdateTransferRequest{ + ExpiresInSeconds: &expirySeconds, + CiphertextBase64: "not-base64", + }) + requireHTTPError(t, err, 400, "invalid_manifest") + }) + + t.Run("repository error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + updateErr: errors.New("update failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + expirySeconds := 3600 + + err := svc.UpdateTransfer(context.Background(), create.TransferID, create.ManageToken, UpdateTransferRequest{ + ExpiresInSeconds: &expirySeconds, + }) + require.ErrorContains(t, err, "update transfer") + }) + + t.Run("invalid expiry", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + expirySeconds := 2 * 3600 + + err := svc.UpdateTransfer(context.Background(), create.TransferID, create.ManageToken, UpdateTransferRequest{ + ExpiresInSeconds: &expirySeconds, + }) + requireHTTPError(t, err, 400, "invalid_expiry") + }) + + t.Run("storage error", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &failingStorage{ + memoryStorage: &memoryStorage{}, + putObjectErr: errors.New("upload failed"), + } + svc := New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + ciphertext := base64.StdEncoding.EncodeToString([]byte("updated-manifest")) + + err := svc.UpdateTransfer(context.Background(), create.TransferID, create.ManageToken, UpdateTransferRequest{ + CiphertextBase64: ciphertext, + }) + require.ErrorContains(t, err, "put updated manifest") + }) + + t.Run("no-op update", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + err := svc.UpdateTransfer(context.Background(), create.TransferID, create.ManageToken, UpdateTransferRequest{}) + require.NoError(t, err) + }) +} + +func TestDeleteTransferHandlesRepositoryAndStorageFailures(t *testing.T) { + t.Parallel() + + t.Run("repository error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + markDeletedErr: errors.New("delete failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + + err := svc.DeleteTransfer(context.Background(), create.TransferID, create.ManageToken) + require.ErrorContains(t, err, "delete transfer") + }) + + t.Run("storage delete prefix errors are ignored", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &failingStorage{ + memoryStorage: &memoryStorage{}, + deletePrefixErr: errors.New("delete prefix failed"), + } + svc := New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + + err := svc.DeleteTransfer(context.Background(), create.TransferID, create.ManageToken) + require.NoError(t, err) + + transfer, getErr := repository.GetTransfer(context.Background(), create.TransferID) + require.NoError(t, getErr) + require.Equal(t, models.TransferStatusDeleted, transfer.Status) + }) + + t.Run("missing token", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + err := svc.DeleteTransfer(context.Background(), create.TransferID, "") + requireHTTPError(t, err, 401, "missing_manage_token") + }) +} + +func TestRegisterFilesPropagatesRepositoryErrors(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + registerFilesErr: errors.New("insert files failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + + err := svc.RegisterFiles(context.Background(), create.TransferID, create.ManageToken, []RegisterFileRequest{{ + FileID: "file-a", + TotalChunks: 1, + CiphertextBytes: 32, + ChunkSize: 32, + }}) + require.ErrorContains(t, err, "register files") +} + +func TestGetPublicTransferHandlesLimiterAndPresignFailures(t *testing.T) { + t.Parallel() + + t.Run("limiter error", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := New(testConfig(), repository, &memoryStorage{}, errorLimiter{err: errors.New("redis unavailable")}) + + _, err := svc.GetPublicTransfer(context.Background(), "198.51.100.10", "transfer-id") + require.ErrorContains(t, err, "rate limit check") + }) + + t.Run("presign error", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &failingStorage{ + memoryStorage: &memoryStorage{}, + presignDownloadErr: errors.New("presign download failed"), + } + svc := New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + makeReadyTransferForTest(t, svc, create.TransferID, create.ManageToken) + + _, err := svc.GetPublicTransfer(context.Background(), "198.51.100.10", create.TransferID) + require.ErrorContains(t, err, "presign manifest download") + }) + + t.Run("repository error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + getTransferErr: errors.New("lookup failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + + _, err := svc.GetPublicTransfer(context.Background(), "198.51.100.10", "transfer-id") + require.ErrorContains(t, err, "lookup failed") + }) +} + +func TestCreateDownloadURLsHandlesNotFoundAndPresignFailures(t *testing.T) { + t.Parallel() + + t.Run("not found", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + getResumeStateErr: repo.ErrNotFound, + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + + _, err := svc.CreateDownloadURLs(context.Background(), "198.51.100.10", "missing-transfer", DownloadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + requireHTTPError(t, err, 404, "not_found") + }) + + t.Run("presign error", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &failingStorage{ + memoryStorage: &memoryStorage{}, + presignDownloadErr: errors.New("presign download failed"), + } + svc := New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) + create := createTransferForTest(t, svc) + makeReadyTransferForTest(t, svc, create.TransferID, create.ManageToken) + + _, err := svc.CreateDownloadURLs(context.Background(), "198.51.100.10", create.TransferID, DownloadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + require.ErrorContains(t, err, "presign chunk download") + }) + + t.Run("limiter error", func(t *testing.T) { + t.Parallel() + + svc := New(testConfig(), newMemoryRepository(), &memoryStorage{}, errorLimiter{err: errors.New("redis unavailable")}) + + _, err := svc.CreateDownloadURLs(context.Background(), "198.51.100.10", "transfer-id", DownloadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + require.ErrorContains(t, err, "rate limit check") + }) + + t.Run("repository error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + getResumeStateErr: errors.New("resume failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + + _, err := svc.CreateDownloadURLs(context.Background(), "198.51.100.10", "transfer-id", DownloadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + require.ErrorContains(t, err, "resume failed") + }) + + t.Run("requested file is unavailable", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + create := createTransferForTest(t, svc) + makeReadyTransferForTest(t, svc, create.TransferID, create.ManageToken) + + _, err := svc.CreateDownloadURLs(context.Background(), "198.51.100.10", create.TransferID, DownloadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "missing-file", ChunkIndex: 0}}, + }) + requireHTTPError(t, err, 400, "invalid_download_request") + }) +} + +func TestCleanupExpiredHandlesRepositoryAndStorageFailures(t *testing.T) { + t.Parallel() + + t.Run("list cleanup candidates error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + listCleanupErr: errors.New("list failed"), + } + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + + err := svc.CleanupExpired(context.Background()) + require.ErrorContains(t, err, "list cleanup candidates") + }) + + t.Run("delete prefix error", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + expired := createExpiredTransferForCleanup() + require.NoError(t, repository.CreateOrReplace(expired)) + + storage := &failingStorage{ + memoryStorage: &memoryStorage{}, + deletePrefixErr: errors.New("delete failed"), + } + svc := New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) + + err := svc.CleanupExpired(context.Background()) + require.ErrorContains(t, err, "delete transfer objects") + }) + + t.Run("mark purged error", func(t *testing.T) { + t.Parallel() + + repository := &failingRepository{ + memoryRepository: newMemoryRepository(), + markPurgedErr: errors.New("mark purged failed"), + } + expired := createExpiredTransferForCleanup() + require.NoError(t, repository.CreateOrReplace(expired)) + + svc := New(testConfig(), repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + + err := svc.CleanupExpired(context.Background()) + require.ErrorContains(t, err, "mark purged") + }) +} + +func createExpiredTransferForCleanup() models.Transfer { + return models.Transfer{ + ID: "expired-for-cleanup", + Status: models.TransferStatusReady, + ManageTokenHash: hashToken("manage-token"), + CreatedAt: time.Now().UTC().Add(-48 * time.Hour), + UpdatedAt: time.Now().UTC().Add(-48 * time.Hour), + ExpiresAt: time.Now().UTC().Add(-time.Hour), + } +} + +type errorLimiter struct { + err error +} + +func (l errorLimiter) Allow(context.Context, string, int, time.Duration) (bool, error) { + return false, l.err +} + +type failingRepository struct { + *memoryRepository + createTransferErr error + getTransferErr error + registerFilesErr error + listFilesErr error + completeChunksErr error + getResumeStateErr error + setManifestErr error + finalizeErr error + updateErr error + markDeletedErr error + listCleanupErr error + markPurgedErr error +} + +func (r *failingRepository) CreateTransfer(ctx context.Context, transfer models.Transfer) error { + if r.createTransferErr != nil { + return r.createTransferErr + } + return r.memoryRepository.CreateTransfer(ctx, transfer) +} + +func (r *failingRepository) GetTransfer(ctx context.Context, transferID string) (models.Transfer, error) { + if r.getTransferErr != nil { + return models.Transfer{}, r.getTransferErr + } + return r.memoryRepository.GetTransfer(ctx, transferID) +} + +func (r *failingRepository) RegisterFiles(ctx context.Context, transferID string, files []models.TransferFile) error { + if r.registerFilesErr != nil { + return r.registerFilesErr + } + return r.memoryRepository.RegisterFiles(ctx, transferID, files) +} + +func (r *failingRepository) ListFiles(ctx context.Context, transferID string) ([]models.TransferFile, error) { + if r.listFilesErr != nil { + return nil, r.listFilesErr + } + return r.memoryRepository.ListFiles(ctx, transferID) +} + +func (r *failingRepository) CompleteChunks(ctx context.Context, transferID string, chunks []models.TransferChunk) error { + if r.completeChunksErr != nil { + return r.completeChunksErr + } + return r.memoryRepository.CompleteChunks(ctx, transferID, chunks) +} + +func (r *failingRepository) GetResumeState(ctx context.Context, transferID string) (models.TransferResumeState, error) { + if r.getResumeStateErr != nil { + return models.TransferResumeState{}, r.getResumeStateErr + } + return r.memoryRepository.GetResumeState(ctx, transferID) +} + +func (r *failingRepository) SetManifest(ctx context.Context, transferID string, objectKey string, ciphertextSize int64) error { + if r.setManifestErr != nil { + return r.setManifestErr + } + return r.memoryRepository.SetManifest(ctx, transferID, objectKey, ciphertextSize) +} + +func (r *failingRepository) FinalizeTransfer(ctx context.Context, transferID string, wrappedRootKey string, totalFiles int, totalCiphertextBytes int64) error { + if r.finalizeErr != nil { + return r.finalizeErr + } + return r.memoryRepository.FinalizeTransfer(ctx, transferID, wrappedRootKey, totalFiles, totalCiphertextBytes) +} + +func (r *failingRepository) UpdateTransfer(ctx context.Context, transferID string, params models.UpdateTransferParams) error { + if r.updateErr != nil { + return r.updateErr + } + return r.memoryRepository.UpdateTransfer(ctx, transferID, params) +} + +func (r *failingRepository) MarkDeleted(ctx context.Context, transferID string) error { + if r.markDeletedErr != nil { + return r.markDeletedErr + } + return r.memoryRepository.MarkDeleted(ctx, transferID) +} + +func (r *failingRepository) ListCleanupCandidates(ctx context.Context, limit int) ([]models.Transfer, error) { + if r.listCleanupErr != nil { + return nil, r.listCleanupErr + } + return r.memoryRepository.ListCleanupCandidates(ctx, limit) +} + +func (r *failingRepository) MarkPurged(ctx context.Context, transferID string) error { + if r.markPurgedErr != nil { + return r.markPurgedErr + } + return r.memoryRepository.MarkPurged(ctx, transferID) +} + +type failingStorage struct { + *memoryStorage + presignUploadErr error + presignDownloadErr error + putObjectErr error + deletePrefixErr error +} + +func (s *failingStorage) PresignUpload(ctx context.Context, objectKey string, ttl time.Duration) (string, error) { + if s.presignUploadErr != nil { + return "", s.presignUploadErr + } + return s.memoryStorage.PresignUpload(ctx, objectKey, ttl) +} + +func (s *failingStorage) PresignDownload(ctx context.Context, objectKey string, ttl time.Duration) (string, error) { + if s.presignDownloadErr != nil { + return "", s.presignDownloadErr + } + return s.memoryStorage.PresignDownload(ctx, objectKey, ttl) +} + +func (s *failingStorage) PutObject(ctx context.Context, objectKey string, body []byte, contentType string) error { + if s.putObjectErr != nil { + return s.putObjectErr + } + return s.memoryStorage.PutObject(ctx, objectKey, body, contentType) +} + +func (s *failingStorage) DeletePrefix(ctx context.Context, prefix string) error { + if s.deletePrefixErr != nil { + return s.deletePrefixErr + } + return s.memoryStorage.DeletePrefix(ctx, prefix) +} + +func (s *failingStorage) EnsureBucket(context.Context) error { + return nil +} diff --git a/apps/api/internal/service/transfer_service_edge_test.go b/apps/api/internal/service/transfer_service_edge_test.go new file mode 100644 index 0000000..969f04f --- /dev/null +++ b/apps/api/internal/service/transfer_service_edge_test.go @@ -0,0 +1,408 @@ +package service + +import ( + "context" + "encoding/base64" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/models" +) + +func TestGetManageTransferRejectsMissingInvalidAndUnknownTokens(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + _, err := svc.GetManageTransfer(context.Background(), create.TransferID, "") + requireHTTPError(t, err, 401, "missing_manage_token") + + _, err = svc.GetManageTransfer(context.Background(), create.TransferID, "wrong-token") + requireHTTPError(t, err, 403, "invalid_manage_token") + + _, err = svc.GetManageTransfer(context.Background(), "missing-transfer", "present-token") + requireHTTPError(t, err, 404, "not_found") +} + +func TestRegisterFilesRejectsInvalidInputs(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + files []RegisterFileRequest + code string + }{ + { + name: "empty files", + files: nil, + code: "empty_files", + }, + { + name: "too many files", + files: func() []RegisterFileRequest { + files := make([]RegisterFileRequest, testConfig().MaxFileCount+1) + for i := range files { + files[i] = RegisterFileRequest{ + FileID: "file", + TotalChunks: 1, + CiphertextBytes: 1, + ChunkSize: 1, + } + } + return files + }(), + code: "too_many_files", + }, + { + name: "blank file id", + files: []RegisterFileRequest{{ + FileID: " ", + TotalChunks: 1, + CiphertextBytes: 1, + ChunkSize: 1, + }}, + code: "invalid_file_registration", + }, + { + name: "zero chunk size", + files: []RegisterFileRequest{{ + FileID: "file-a", + TotalChunks: 1, + CiphertextBytes: 1, + ChunkSize: 0, + }}, + code: "invalid_file_registration", + }, + { + name: "transfer too large", + files: []RegisterFileRequest{{ + FileID: "file-a", + TotalChunks: 1, + CiphertextBytes: testConfig().MaxTransferBytes + 1, + ChunkSize: 1, + }}, + code: "transfer_too_large", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + err := svc.RegisterFiles(context.Background(), create.TransferID, create.ManageToken, tc.files) + requireHTTPError(t, err, 400, tc.code) + }) + } +} + +func TestRegisterFilesRejectsExpiredTransfer(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + expireTransfer(t, repository, create.TransferID) + + err := svc.RegisterFiles(context.Background(), create.TransferID, create.ManageToken, []RegisterFileRequest{{ + FileID: "file-a", + TotalChunks: 1, + CiphertextBytes: 1, + ChunkSize: 1, + }}) + requireHTTPError(t, err, 410, "expired") +} + +func TestCreateUploadURLsRejectsUnknownAndOutOfRangeChunks(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + + testCases := []struct { + name string + chunk UploadChunkRequest + }{ + { + name: "unknown file", + chunk: UploadChunkRequest{FileID: "missing-file", ChunkIndex: 0}, + }, + { + name: "negative index", + chunk: UploadChunkRequest{FileID: "file-a", ChunkIndex: -1}, + }, + { + name: "past total chunks", + chunk: UploadChunkRequest{FileID: "file-a", ChunkIndex: 2}, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := svc.CreateUploadURLs(context.Background(), create.TransferID, create.ManageToken, UploadURLRequest{ + Chunks: []UploadChunkRequest{tc.chunk}, + }) + requireHTTPError(t, err, 400, "invalid_chunk_request") + }) + } +} + +func TestPutManifestRejectsInvalidCiphertext(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + err := svc.PutManifest(context.Background(), create.TransferID, create.ManageToken, ManifestUploadRequest{ + CiphertextBase64: "not-base64", + }) + requireHTTPError(t, err, 400, "invalid_manifest") + + err = svc.PutManifest(context.Background(), create.TransferID, create.ManageToken, ManifestUploadRequest{ + CiphertextBase64: "", + }) + requireHTTPError(t, err, 400, "invalid_manifest") +} + +func TestFinalizeTransferRejectsInvalidPayload(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + create := createTransferForTest(t, svc) + registerFileForTest(t, svc, create.TransferID, create.ManageToken) + + err := svc.FinalizeTransfer(context.Background(), create.TransferID, create.ManageToken, FinalizeTransferRequest{ + WrappedRootKey: "", + TotalFiles: 1, + TotalCiphertextBytes: 16, + }) + requireHTTPError(t, err, 400, "invalid_finalize") +} + +func TestUpdateTransferUpdatesExpiryAndManifest(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + create := createTransferForTest(t, svc) + + transfer, err := repository.GetTransfer(context.Background(), create.TransferID) + require.NoError(t, err) + transfer.ManifestObjectKey = "transfers/custom/manifest.bin" + require.NoError(t, repository.CreateOrReplace(transfer)) + + ciphertext := []byte(`{"version":2}`) + expirySeconds := 3 * 3600 + err = svc.UpdateTransfer(context.Background(), create.TransferID, create.ManageToken, UpdateTransferRequest{ + ExpiresInSeconds: &expirySeconds, + CiphertextBase64: base64.StdEncoding.EncodeToString(ciphertext), + }) + require.NoError(t, err) + + updated, err := repository.GetTransfer(context.Background(), create.TransferID) + require.NoError(t, err) + require.Equal(t, int64(len(ciphertext)), updated.ManifestCiphertextSize) + require.WithinDuration(t, time.Now().UTC().Add(3*time.Hour), updated.ExpiresAt, 3*time.Second) + require.Equal(t, ciphertext, storage.objects["transfers/custom/manifest.bin"]) +} + +func TestUpdateTransferGeneratesManifestObjectKeyWhenMissing(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + create := createTransferForTest(t, svc) + + ciphertext := []byte(`{"renamed":true}`) + err := svc.UpdateTransfer(context.Background(), create.TransferID, create.ManageToken, UpdateTransferRequest{ + CiphertextBase64: base64.StdEncoding.EncodeToString(ciphertext), + }) + require.NoError(t, err) + + updated, err := repository.GetTransfer(context.Background(), create.TransferID) + require.NoError(t, err) + require.Equal(t, manifestObjectKey(create.TransferID), updated.ManifestObjectKey) + require.Equal(t, int64(len(ciphertext)), updated.ManifestCiphertextSize) + require.Equal(t, ciphertext, storage.objects[manifestObjectKey(create.TransferID)]) +} + +func TestUpdateTransferRejectsDeletedTransfer(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + transfer, err := repository.GetTransfer(context.Background(), create.TransferID) + require.NoError(t, err) + now := time.Now().UTC() + transfer.Status = models.TransferStatusDeleted + transfer.DeletedAt = &now + require.NoError(t, repository.CreateOrReplace(transfer)) + + expirySeconds := 3600 + err = svc.UpdateTransfer(context.Background(), create.TransferID, create.ManageToken, UpdateTransferRequest{ + ExpiresInSeconds: &expirySeconds, + }) + requireHTTPError(t, err, 410, "deleted") +} + +func TestGetPublicTransferReturnsNotFoundAndExpiredStatuses(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + + _, err := svc.GetPublicTransfer(context.Background(), "198.51.100.20", "missing-transfer") + requireHTTPError(t, err, 404, "not_found") + + create := createTransferForTest(t, svc) + transfer, err := repository.GetTransfer(context.Background(), create.TransferID) + require.NoError(t, err) + transfer.Status = models.TransferStatusReady + transfer.ManifestObjectKey = manifestObjectKey(create.TransferID) + transfer.WrappedRootKey = `{"version":1}` + transfer.ExpiresAt = time.Now().UTC().Add(-time.Minute) + require.NoError(t, repository.CreateOrReplace(transfer)) + + response, err := svc.GetPublicTransfer(context.Background(), "198.51.100.20", create.TransferID) + require.NoError(t, err) + require.Equal(t, "expired", response.Status) + require.Empty(t, response.ManifestURL) + require.Empty(t, response.WrappedRootKey) +} + +func TestCreateDownloadURLsValidatesAvailabilityAndChunkPresence(t *testing.T) { + t.Parallel() + + t.Run("transfer not ready", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + create := createTransferForTest(t, svc) + + _, err := svc.CreateDownloadURLs(context.Background(), "198.51.100.20", create.TransferID, DownloadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 0}}, + }) + requireHTTPError(t, err, 409, "transfer_unavailable") + }) + + t.Run("invalid chunk request", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + create := createTransferForTest(t, svc) + makeReadyTransferForTest(t, svc, create.TransferID, create.ManageToken) + + _, err := svc.CreateDownloadURLs(context.Background(), "198.51.100.20", create.TransferID, DownloadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 99}}, + }) + requireHTTPError(t, err, 400, "invalid_download_request") + }) + + t.Run("returns presigned url for uploaded chunk", func(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + create := createTransferForTest(t, svc) + makeReadyTransferForTest(t, svc, create.TransferID, create.ManageToken) + + items, err := svc.CreateDownloadURLs(context.Background(), "198.51.100.20", create.TransferID, DownloadURLRequest{ + Chunks: []UploadChunkRequest{{FileID: "file-a", ChunkIndex: 1}}, + }) + require.NoError(t, err) + require.Len(t, items, 1) + require.Equal(t, "file-a", items[0].FileID) + require.Equal(t, 1, items[0].ChunkIndex) + require.Contains(t, items[0].URL, chunkObjectKey(create.TransferID, "file-a", 1)) + }) +} + +func createTransferForTest(t *testing.T, svc *Service) CreateTransferResponse { + t.Helper() + + response, err := svc.CreateTransfer(context.Background(), "198.51.100.10", CreateTransferRequest{ + ExpiresInSeconds: 3600, + }) + require.NoError(t, err) + return response +} + +func registerFileForTest(t *testing.T, svc *Service, transferID string, manageToken string) { + t.Helper() + + err := svc.RegisterFiles(context.Background(), transferID, manageToken, []RegisterFileRequest{{ + FileID: "file-a", + TotalChunks: 2, + CiphertextBytes: 64, + PlaintextBytes: int64ptr(32), + ChunkSize: 32, + }}) + require.NoError(t, err) +} + +func makeReadyTransferForTest(t *testing.T, svc *Service, transferID string, manageToken string) { + t.Helper() + + registerFileForTest(t, svc, transferID, manageToken) + + err := svc.CompleteChunks(context.Background(), transferID, manageToken, []CompleteChunkRequest{ + {FileID: "file-a", ChunkIndex: 0, CiphertextSize: 32, ChecksumSHA256: "a"}, + {FileID: "file-a", ChunkIndex: 1, CiphertextSize: 32, ChecksumSHA256: "b"}, + }) + require.NoError(t, err) + + err = svc.PutManifest(context.Background(), transferID, manageToken, ManifestUploadRequest{ + CiphertextBase64: base64.StdEncoding.EncodeToString([]byte(`{"version":1}`)), + }) + require.NoError(t, err) + + err = svc.FinalizeTransfer(context.Background(), transferID, manageToken, FinalizeTransferRequest{ + WrappedRootKey: `{"version":1}`, + TotalFiles: 1, + TotalCiphertextBytes: 64, + }) + require.NoError(t, err) +} + +func expireTransfer(t *testing.T, repository *memoryRepository, transferID string) { + t.Helper() + + transfer, err := repository.GetTransfer(context.Background(), transferID) + require.NoError(t, err) + transfer.ExpiresAt = time.Now().UTC().Add(-time.Minute) + require.NoError(t, repository.CreateOrReplace(transfer)) +} + +func requireHTTPError(t *testing.T, err error, status int, code string) { + t.Helper() + + require.Error(t, err) + + httpErr, ok := err.(*HTTPError) + require.True(t, ok) + require.Equal(t, status, httpErr.Status) + require.Equal(t, code, httpErr.Code) +} diff --git a/apps/api/internal/service/transfer_service_test.go b/apps/api/internal/service/transfer_service_test.go new file mode 100644 index 0000000..f4ad1a2 --- /dev/null +++ b/apps/api/internal/service/transfer_service_test.go @@ -0,0 +1,374 @@ +package service + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/xdrop/monorepo/internal/config" + "github.com/xdrop/monorepo/internal/models" + "github.com/xdrop/monorepo/internal/ratelimit" + "github.com/xdrop/monorepo/internal/repo" +) + +func TestCreateTransferStoresHashedManageToken(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + + response, err := svc.CreateTransfer(context.Background(), "127.0.0.1", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.NoError(t, err) + require.NotEmpty(t, response.TransferID) + require.NotEmpty(t, response.ManageToken) + + transfer, err := repository.GetTransfer(context.Background(), response.TransferID) + require.NoError(t, err) + require.NotEqual(t, response.ManageToken, transfer.ManageTokenHash) + require.Len(t, transfer.ManageTokenHash, 64) +} + +func TestCreateTransferRejectsUnsupportedExpiry(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + svc := newTestService(repository) + + _, err := svc.CreateTransfer(context.Background(), "127.0.0.1", CreateTransferRequest{ExpiresInSeconds: 2 * 3600}) + require.Error(t, err) + + httpErr, ok := err.(*HTTPError) + require.True(t, ok) + require.Equal(t, 400, httpErr.Status) + require.Equal(t, "invalid_expiry", httpErr.Code) +} + +func TestFinalizeFlowPublishesDescriptor(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + + create, err := svc.CreateTransfer(context.Background(), "127.0.0.1", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.NoError(t, err) + + err = svc.RegisterFiles(context.Background(), create.TransferID, create.ManageToken, []RegisterFileRequest{ + { + FileID: "file_a", + TotalChunks: 2, + CiphertextBytes: 64, + PlaintextBytes: int64ptr(32), + ChunkSize: 16, + }, + }) + require.NoError(t, err) + + err = svc.CompleteChunks(context.Background(), create.TransferID, create.ManageToken, []CompleteChunkRequest{ + {FileID: "file_a", ChunkIndex: 0, CiphertextSize: 32, ChecksumSHA256: strings.Repeat("a", 64)}, + {FileID: "file_a", ChunkIndex: 1, CiphertextSize: 32, ChecksumSHA256: strings.Repeat("b", 64)}, + }) + require.NoError(t, err) + + ciphertext := base64.StdEncoding.EncodeToString([]byte(`{"version":1}`)) + err = svc.PutManifest(context.Background(), create.TransferID, create.ManageToken, ManifestUploadRequest{ + CiphertextBase64: ciphertext, + }) + require.NoError(t, err) + + err = svc.FinalizeTransfer(context.Background(), create.TransferID, create.ManageToken, FinalizeTransferRequest{ + WrappedRootKey: `{"version":1}`, + TotalFiles: 1, + TotalCiphertextBytes: 64, + }) + require.NoError(t, err) + + publicDescriptor, err := svc.GetPublicTransfer(context.Background(), "127.0.0.1", create.TransferID) + require.NoError(t, err) + require.Equal(t, "ready", publicDescriptor.Status) + require.NotEmpty(t, publicDescriptor.ManifestURL) + require.Equal(t, `{"version":1}`, publicDescriptor.WrappedRootKey) +} + +func TestRateLimitBlocksSecondCreate(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + cfg := testConfig() + cfg.CreateLimit = 1 + svc := New(cfg, repository, &memoryStorage{}, ratelimit.NewMemoryLimiter()) + + _, err := svc.CreateTransfer(context.Background(), "shared-ip", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.NoError(t, err) + + _, err = svc.CreateTransfer(context.Background(), "shared-ip", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.Error(t, err) + + httpErr, ok := err.(*HTTPError) + require.True(t, ok) + require.Equal(t, 429, httpErr.Status) +} + +func TestDeleteMakesTransferUnavailable(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + + create, err := svc.CreateTransfer(context.Background(), "127.0.0.1", CreateTransferRequest{ExpiresInSeconds: 3600}) + require.NoError(t, err) + + readyTransfer, err := repository.GetTransfer(context.Background(), create.TransferID) + require.NoError(t, err) + readyTransfer.Status = models.TransferStatusReady + readyTransfer.ManifestObjectKey = "transfers/demo/manifest.bin" + readyTransfer.WrappedRootKey = `{"version":1}` + require.NoError(t, repository.CreateOrReplace(readyTransfer)) + + err = svc.DeleteTransfer(context.Background(), create.TransferID, create.ManageToken) + require.NoError(t, err) + + publicDescriptor, err := svc.GetPublicTransfer(context.Background(), "127.0.0.1", create.TransferID) + require.NoError(t, err) + require.Equal(t, "deleted", publicDescriptor.Status) +} + +func TestCleanupRemovesExpiredPrefixes(t *testing.T) { + t.Parallel() + + repository := newMemoryRepository() + storage := &memoryStorage{} + svc := newTestServiceWithStorage(repository, storage) + + expired := models.Transfer{ + ID: "expired-transfer", + Status: models.TransferStatusReady, + ManageTokenHash: hashToken("manage-token"), + CreatedAt: time.Now().Add(-48 * time.Hour), + UpdatedAt: time.Now().Add(-48 * time.Hour), + ExpiresAt: time.Now().Add(-24 * time.Hour), + ManifestObjectKey: "transfers/expired-transfer/manifest.bin", + WrappedRootKey: `{"version":1}`, + } + require.NoError(t, repository.CreateOrReplace(expired)) + + err := svc.CleanupExpired(context.Background()) + require.NoError(t, err) + require.Contains(t, storage.deletedPrefixes, "transfers/expired-transfer/") + + transfer, err := repository.GetTransfer(context.Background(), expired.ID) + require.NoError(t, err) + require.NotNil(t, transfer.PurgedAt) +} + +type memoryRepository struct { + transfers map[string]models.Transfer + files map[string][]models.TransferFile + chunks map[string][]models.TransferChunk +} + +func newMemoryRepository() *memoryRepository { + return &memoryRepository{ + transfers: map[string]models.Transfer{}, + files: map[string][]models.TransferFile{}, + chunks: map[string][]models.TransferChunk{}, + } +} + +func (m *memoryRepository) CreateTransfer(_ context.Context, transfer models.Transfer) error { + m.transfers[transfer.ID] = transfer + return nil +} + +func (m *memoryRepository) CreateOrReplace(transfer models.Transfer) error { + m.transfers[transfer.ID] = transfer + return nil +} + +func (m *memoryRepository) GetTransfer(_ context.Context, transferID string) (models.Transfer, error) { + transfer, ok := m.transfers[transferID] + if !ok { + return models.Transfer{}, repo.ErrNotFound + } + return transfer, nil +} + +func (m *memoryRepository) RegisterFiles(_ context.Context, transferID string, files []models.TransferFile) error { + m.files[transferID] = append([]models.TransferFile{}, files...) + transfer := m.transfers[transferID] + transfer.Status = models.TransferStatusUploading + m.transfers[transferID] = transfer + return nil +} + +func (m *memoryRepository) ListFiles(_ context.Context, transferID string) ([]models.TransferFile, error) { + return append([]models.TransferFile{}, m.files[transferID]...), nil +} + +func (m *memoryRepository) CompleteChunks(_ context.Context, transferID string, chunks []models.TransferChunk) error { + m.chunks[transferID] = append(m.chunks[transferID], chunks...) + files := m.files[transferID] + for fileIndex := range files { + uploaded := 0 + for _, chunk := range m.chunks[transferID] { + if chunk.OpaqueFileID == files[fileIndex].OpaqueFileID { + uploaded++ + } + } + if uploaded >= files[fileIndex].TotalChunks { + files[fileIndex].UploadStatus = "complete" + } + } + m.files[transferID] = files + return nil +} + +func (m *memoryRepository) GetResumeState(ctx context.Context, transferID string) (models.TransferResumeState, error) { + transfer, err := m.GetTransfer(ctx, transferID) + if err != nil { + return models.TransferResumeState{}, err + } + uploaded := map[string][]int{} + for _, chunk := range m.chunks[transferID] { + uploaded[chunk.OpaqueFileID] = append(uploaded[chunk.OpaqueFileID], chunk.ChunkIndex) + } + return models.TransferResumeState{ + Transfer: transfer, + Files: append([]models.TransferFile{}, m.files[transferID]...), + UploadedChunks: uploaded, + }, nil +} + +func (m *memoryRepository) SetManifest(_ context.Context, transferID string, objectKey string, ciphertextSize int64) error { + transfer := m.transfers[transferID] + transfer.ManifestObjectKey = objectKey + transfer.ManifestCiphertextSize = ciphertextSize + m.transfers[transferID] = transfer + return nil +} + +func (m *memoryRepository) FinalizeTransfer(_ context.Context, transferID string, wrappedRootKey string, totalFiles int, totalCiphertextBytes int64) error { + transfer := m.transfers[transferID] + if transfer.ManifestObjectKey == "" { + return fmt.Errorf("manifest missing") + } + for _, file := range m.files[transferID] { + if file.UploadStatus != "complete" { + return fmt.Errorf("upload incomplete") + } + } + transfer.Status = models.TransferStatusReady + transfer.WrappedRootKey = wrappedRootKey + transfer.TotalFiles = totalFiles + transfer.TotalCiphertextBytes = totalCiphertextBytes + now := time.Now() + transfer.FinalizedAt = &now + m.transfers[transferID] = transfer + return nil +} + +func (m *memoryRepository) UpdateTransfer(_ context.Context, transferID string, params models.UpdateTransferParams) error { + transfer := m.transfers[transferID] + if params.ManifestObjectKey != nil { + transfer.ManifestObjectKey = *params.ManifestObjectKey + } + if params.ExpiresAt != nil { + transfer.ExpiresAt = *params.ExpiresAt + } + if params.ManifestCiphertextSize != nil { + transfer.ManifestCiphertextSize = *params.ManifestCiphertextSize + } + m.transfers[transferID] = transfer + return nil +} + +func (m *memoryRepository) MarkDeleted(_ context.Context, transferID string) error { + transfer := m.transfers[transferID] + now := time.Now() + transfer.Status = models.TransferStatusDeleted + transfer.DeletedAt = &now + m.transfers[transferID] = transfer + return nil +} + +func (m *memoryRepository) ListCleanupCandidates(_ context.Context, _ int) ([]models.Transfer, error) { + candidates := []models.Transfer{} + for _, transfer := range m.transfers { + if transfer.PurgedAt != nil { + continue + } + if transfer.DeletedAt != nil || transfer.ExpiresAt.Before(time.Now()) { + candidates = append(candidates, transfer) + } + } + return candidates, nil +} + +func (m *memoryRepository) MarkPurged(_ context.Context, transferID string) error { + transfer := m.transfers[transferID] + now := time.Now() + transfer.PurgedAt = &now + transfer.Status = models.TransferStatusExpired + m.transfers[transferID] = transfer + return nil +} + +type memoryStorage struct { + objects map[string][]byte + deletedPrefixes []string +} + +func (m *memoryStorage) PresignUpload(_ context.Context, objectKey string, _ time.Duration) (string, error) { + return "https://example.test/upload/" + objectKey, nil +} + +func (m *memoryStorage) PresignDownload(_ context.Context, objectKey string, _ time.Duration) (string, error) { + return "https://example.test/download/" + objectKey, nil +} + +func (m *memoryStorage) PutObject(_ context.Context, objectKey string, body []byte, _ string) error { + if m.objects == nil { + m.objects = map[string][]byte{} + } + m.objects[objectKey] = append([]byte{}, body...) + return nil +} + +func (m *memoryStorage) DeletePrefix(_ context.Context, prefix string) error { + m.deletedPrefixes = append(m.deletedPrefixes, prefix) + return nil +} + +func (m *memoryStorage) EnsureBucket(_ context.Context) error { + return nil +} + +func newTestService(repository *memoryRepository) *Service { + return newTestServiceWithStorage(repository, &memoryStorage{}) +} + +func newTestServiceWithStorage(repository *memoryRepository, storage *memoryStorage) *Service { + return New(testConfig(), repository, storage, ratelimit.NewMemoryLimiter()) +} + +func testConfig() config.Config { + return config.Config{ + ChunkSize: 8 * 1024 * 1024, + DefaultExpiry: time.Hour, + CreateLimit: 20, + PublicReadLimit: 120, + DownloadURLLimit: 120, + PresignTTL: 5 * time.Minute, + MaxFileCount: 100, + MaxTransferBytes: 256 * 1024 * 1024, + } +} + +func int64ptr(value int64) *int64 { + return &value +} diff --git a/apps/api/internal/storage/s3.go b/apps/api/internal/storage/s3.go new file mode 100644 index 0000000..27a1d0c --- /dev/null +++ b/apps/api/internal/storage/s3.go @@ -0,0 +1,246 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +// ObjectStorage defines the storage operations the service layer needs for transfer objects. +type ObjectStorage interface { + PresignUpload(ctx context.Context, objectKey string, ttl time.Duration) (string, error) + PresignDownload(ctx context.Context, objectKey string, ttl time.Duration) (string, error) + PutObject(ctx context.Context, objectKey string, body []byte, contentType string) error + DeletePrefix(ctx context.Context, prefix string) error + EnsureBucket(ctx context.Context) error +} + +type s3Client interface { + s3.ListObjectsV2APIClient + CreateBucket(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) + DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + HeadBucket(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) +} + +type s3Presigner interface { + PresignGetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) + PresignPutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) +} + +type s3Uploader interface { + Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) +} + +type listObjectsV2Paginator interface { + HasMorePages() bool + NextPage(ctx context.Context, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) +} + +// S3Storage stores transfer objects in an S3-compatible bucket. +type S3Storage struct { + bucket string + client s3Client + presigner s3Presigner + uploader s3Uploader + paginator func(client s3.ListObjectsV2APIClient, input *s3.ListObjectsV2Input) listObjectsV2Paginator +} + +// Config describes how to connect to the private and public S3-compatible endpoints. +type Config struct { + Endpoint string + PublicEndpoint string + Region string + Bucket string + AccessKey string + SecretKey string + UseSSL bool +} + +var loadDefaultAWSConfig = awsconfig.LoadDefaultConfig + +// NewS3Storage builds an S3-backed object storage adapter with optional public presign endpoint. +func NewS3Storage(ctx context.Context, cfg Config) (*S3Storage, error) { + endpoint := cfg.Endpoint + awsCfg, err := loadDefaultAWSConfig( + ctx, + awsconfig.WithRegion(cfg.Region), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "")), + awsconfig.WithBaseEndpoint(endpoint), + ) + if err != nil { + return nil, fmt.Errorf("load aws config: %w", err) + } + + client := s3.NewFromConfig(awsCfg, func(options *s3.Options) { + options.UsePathStyle = true + }) + presignClient := client + + if cfg.PublicEndpoint != "" && cfg.PublicEndpoint != cfg.Endpoint { + publicCfg, configErr := loadDefaultAWSConfig( + ctx, + awsconfig.WithRegion(cfg.Region), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "")), + awsconfig.WithBaseEndpoint(cfg.PublicEndpoint), + ) + if configErr != nil { + return nil, fmt.Errorf("load public aws config: %w", configErr) + } + presignClient = s3.NewFromConfig(publicCfg, func(options *s3.Options) { + options.UsePathStyle = true + }) + } + + return &S3Storage{ + bucket: cfg.Bucket, + client: client, + presigner: s3.NewPresignClient(presignClient), + uploader: manager.NewUploader(client), + paginator: func(client s3.ListObjectsV2APIClient, input *s3.ListObjectsV2Input) listObjectsV2Paginator { + return s3.NewListObjectsV2Paginator(client, input) + }, + }, nil +} + +// EnsureBucket creates the bucket if it does not already exist. +func (s *S3Storage) EnsureBucket(ctx context.Context) error { + _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)}) + if err == nil { + return nil + } + + _, err = s.client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(s.bucket), + }) + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "bucketalreadyownedbyyou") { + return fmt.Errorf("create bucket: %w", err) + } + + return nil +} + +// PresignUpload returns a time-limited PUT URL for one object key. +func (s *S3Storage) PresignUpload(ctx context.Context, objectKey string, ttl time.Duration) (string, error) { + result, err := s.presigner.PresignPutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(objectKey), + }, s3.WithPresignExpires(ttl)) + if err != nil { + return "", fmt.Errorf("presign upload: %w", err) + } + + return result.URL, nil +} + +// PresignDownload returns a time-limited GET URL for one object key. +func (s *S3Storage) PresignDownload(ctx context.Context, objectKey string, ttl time.Duration) (string, error) { + result, err := s.presigner.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(objectKey), + }, s3.WithPresignExpires(ttl)) + if err != nil { + return "", fmt.Errorf("presign download: %w", err) + } + + return result.URL, nil +} + +// PutObject uploads a complete object payload with the provided content type. +func (s *S3Storage) PutObject(ctx context.Context, objectKey string, body []byte, contentType string) error { + reader := bytes.NewReader(body) + + _, err := s.uploader.Upload(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(objectKey), + Body: reader, + ContentType: aws.String(contentType), + }) + if err != nil { + return fmt.Errorf("upload object: %w", err) + } + + return nil +} + +// DeletePrefix removes every object currently stored beneath a transfer prefix. +func (s *S3Storage) DeletePrefix(ctx context.Context, prefix string) error { + paginator := s.paginator(s.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), + Prefix: aws.String(prefix), + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return fmt.Errorf("list objects: %w", err) + } + + if len(page.Contents) == 0 { + continue + } + + objects := make([]s3typesObjectIdentifier, 0, len(page.Contents)) + for _, item := range page.Contents { + if item.Key == nil { + continue + } + objects = append(objects, s3typesObjectIdentifier{Key: item.Key}) + } + + if len(objects) == 0 { + continue + } + + deleteObjects := make([]s3types.ObjectIdentifier, 0, len(objects)) + for _, object := range objects { + deleteObjects = append(deleteObjects, s3types.ObjectIdentifier{Key: object.Key}) + } + + _, err = s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(s.bucket), + Delete: &s3types.Delete{Objects: deleteObjects}, + }) + if err != nil { + return fmt.Errorf("delete objects: %w", err) + } + } + + return nil +} + +type s3typesObjectIdentifier struct { + Key *string +} + +// NormalizeEndpoint ensures user-supplied endpoints always parse as full URLs. +func NormalizeEndpoint(raw string) (string, error) { + value := strings.TrimSpace(raw) + if !strings.Contains(value, "://") { + value = "http://" + value + } + + parsed, err := url.Parse(value) + if err != nil { + return "", fmt.Errorf("parse endpoint: %w", err) + } + + return parsed.String(), nil +} + +// ReadAll is kept as a shim so tests can stub object reads without importing io directly. +func ReadAll(reader io.Reader) ([]byte, error) { + return io.ReadAll(reader) +} diff --git a/apps/api/internal/storage/s3_integration_test.go b/apps/api/internal/storage/s3_integration_test.go new file mode 100644 index 0000000..cb0fffe --- /dev/null +++ b/apps/api/internal/storage/s3_integration_test.go @@ -0,0 +1,172 @@ +package storage + +import ( + "context" + "fmt" + "os/exec" + "runtime" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func TestS3StorageLifecycleAgainstMinIO(t *testing.T) { + skipIfDockerUnavailable(t) + + ctx := context.Background() + cfg := startMinIOConfig(t, ctx) + + store, err := NewS3Storage(ctx, cfg) + require.NoError(t, err) + + require.NoError(t, store.EnsureBucket(ctx)) + require.NoError(t, store.EnsureBucket(ctx)) + + require.NoError(t, store.PutObject(ctx, "transfers/demo/manifest.bin", []byte("manifest"), "application/octet-stream")) + require.NoError(t, store.PutObject(ctx, "transfers/demo/files/file-a/chunks/00000000.bin", []byte("chunk"), "application/octet-stream")) + require.NoError(t, store.PutObject(ctx, "transfers/other/manifest.bin", []byte("other"), "application/octet-stream")) + + objectBody := getObjectBody(t, ctx, store, "transfers/demo/manifest.bin") + require.Equal(t, []byte("manifest"), objectBody) + + uploadURL, err := store.PresignUpload(ctx, "transfers/demo/files/file-a/chunks/00000001.bin", 5*time.Minute) + require.NoError(t, err) + require.Contains(t, uploadURL, "X-Amz-Algorithm=") + require.Contains(t, uploadURL, "X-Amz-Signature=") + require.Contains(t, uploadURL, cfg.PublicEndpoint) + + downloadURL, err := store.PresignDownload(ctx, "transfers/demo/manifest.bin", 5*time.Minute) + require.NoError(t, err) + require.Contains(t, downloadURL, "X-Amz-Algorithm=") + require.Contains(t, downloadURL, cfg.PublicEndpoint) + + require.NoError(t, store.DeletePrefix(ctx, "transfers/demo/")) + + keys := listObjectKeys(t, ctx, store) + require.NotContains(t, keys, "transfers/demo/manifest.bin") + require.NotContains(t, keys, "transfers/demo/files/file-a/chunks/00000000.bin") + require.Contains(t, keys, "transfers/other/manifest.bin") +} + +func TestNormalizeEndpointHandlesSchemeAndWhitespace(t *testing.T) { + t.Parallel() + + normalized, err := NormalizeEndpoint(" localhost:9000 ") + require.NoError(t, err) + require.Equal(t, "http://localhost:9000", normalized) + + normalized, err = NormalizeEndpoint("https://minio.example.test") + require.NoError(t, err) + require.Equal(t, "https://minio.example.test", normalized) +} + +func TestNormalizeEndpointRejectsInvalidURL(t *testing.T) { + t.Parallel() + + _, err := NormalizeEndpoint("http://%zz") + require.Error(t, err) +} + +func TestReadAllReadsEntireStream(t *testing.T) { + t.Parallel() + + body, err := ReadAll(strings.NewReader("hello")) + require.NoError(t, err) + require.Equal(t, []byte("hello"), body) +} + +func startMinIOConfig(t *testing.T, ctx context.Context) Config { + t.Helper() + + container, err := testcontainers.Run( + ctx, + "minio/minio:latest", + testcontainers.WithEnv(map[string]string{ + "MINIO_ROOT_USER": "minioadmin", + "MINIO_ROOT_PASSWORD": "minioadmin", + }), + testcontainers.WithExposedPorts("9000/tcp"), + testcontainers.WithCmd("server", "/data"), + testcontainers.WithWaitStrategy( + wait.ForHTTP("/minio/health/live"). + WithPort("9000/tcp"). + WithStartupTimeout(90*time.Second), + ), + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, testcontainers.TerminateContainer(container)) + }) + + host, err := container.Host(ctx) + require.NoError(t, err) + port, err := container.MappedPort(ctx, "9000/tcp") + require.NoError(t, err) + + internalEndpoint := fmt.Sprintf("http://%s:%s", host, port.Port()) + return Config{ + Endpoint: internalEndpoint, + PublicEndpoint: "http://public.example.test:9000", + Region: "us-east-1", + Bucket: "xdrop", + AccessKey: "minioadmin", + SecretKey: "minioadmin", + } +} + +func getObjectBody(t *testing.T, ctx context.Context, store *S3Storage, objectKey string) []byte { + t.Helper() + + response, err := store.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(store.bucket), + Key: aws.String(objectKey), + }) + require.NoError(t, err) + defer response.Body.Close() + + body, err := ReadAll(response.Body) + require.NoError(t, err) + return body +} + +func listObjectKeys(t *testing.T, ctx context.Context, store *S3Storage) []string { + t.Helper() + + paginator := s3.NewListObjectsV2Paginator(store.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(store.bucket), + }) + + keys := []string{} + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + require.NoError(t, err) + for _, item := range page.Contents { + if item.Key != nil { + keys = append(keys, *item.Key) + } + } + } + + return keys +} + +func skipIfDockerUnavailable(t *testing.T) { + t.Helper() + + if testing.Short() { + t.Skip("skipping docker-backed integration test in short mode") + } + if runtime.GOOS == "windows" { + t.Skip("skipping docker-backed integration test on windows") + } + + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skipf("skipping docker-backed integration test: %v", err) + } +} diff --git a/apps/api/internal/storage/s3_unit_test.go b/apps/api/internal/storage/s3_unit_test.go new file mode 100644 index 0000000..de59535 --- /dev/null +++ b/apps/api/internal/storage/s3_unit_test.go @@ -0,0 +1,465 @@ +package storage + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/require" +) + +func TestNewS3StoragePresignsAgainstConfiguredEndpoints(t *testing.T) { + ctx := context.Background() + store, err := NewS3Storage(ctx, Config{ + Endpoint: "http://internal.example.test:9000", + PublicEndpoint: "http://public.example.test:9000", + Region: "us-east-1", + Bucket: "xdrop", + AccessKey: "key", + SecretKey: "secret", + }) + require.NoError(t, err) + + uploadURL, err := store.PresignUpload(ctx, "transfers/demo/object.bin", time.Minute) + require.NoError(t, err) + require.Contains(t, uploadURL, "public.example.test:9000") + + downloadURL, err := store.PresignDownload(ctx, "transfers/demo/object.bin", time.Minute) + require.NoError(t, err) + require.Contains(t, downloadURL, "public.example.test:9000") + + internalStore, err := NewS3Storage(ctx, Config{ + Endpoint: "http://internal.example.test:9000", + Region: "us-east-1", + Bucket: "xdrop", + AccessKey: "key", + SecretKey: "secret", + }) + require.NoError(t, err) + + uploadURL, err = internalStore.PresignUpload(ctx, "transfers/demo/object.bin", time.Minute) + require.NoError(t, err) + require.Contains(t, uploadURL, "internal.example.test:9000") + + t.Run("surfaces aws config load failures", func(t *testing.T) { + originalLoader := loadDefaultAWSConfig + t.Cleanup(func() { + loadDefaultAWSConfig = originalLoader + }) + loadDefaultAWSConfig = func(context.Context, ...func(*awsconfig.LoadOptions) error) (aws.Config, error) { + return aws.Config{}, errors.New("boom") + } + + _, err := NewS3Storage(ctx, Config{ + Endpoint: "http://internal.example.test:9000", + Region: "us-east-1", + Bucket: "xdrop", + AccessKey: "key", + SecretKey: "secret", + }) + require.ErrorContains(t, err, "load aws config") + }) + + t.Run("surfaces public aws config load failures", func(t *testing.T) { + originalLoader := loadDefaultAWSConfig + t.Cleanup(func() { + loadDefaultAWSConfig = originalLoader + }) + callCount := 0 + loadDefaultAWSConfig = func(innerCtx context.Context, opts ...func(*awsconfig.LoadOptions) error) (aws.Config, error) { + callCount++ + if callCount == 2 { + return aws.Config{}, errors.New("boom") + } + return originalLoader(innerCtx, opts...) + } + + _, err := NewS3Storage(ctx, Config{ + Endpoint: "http://internal.example.test:9000", + PublicEndpoint: "http://public.example.test:9000", + Region: "us-east-1", + Bucket: "xdrop", + AccessKey: "key", + SecretKey: "secret", + }) + require.ErrorContains(t, err, "load public aws config") + }) + + t.Run("initializes the default paginator factory", func(t *testing.T) { + t.Parallel() + + store, err := NewS3Storage(ctx, Config{ + Endpoint: "http://internal.example.test:9000", + Region: "us-east-1", + Bucket: "xdrop", + AccessKey: "key", + SecretKey: "secret", + }) + require.NoError(t, err) + require.NotNil(t, store.paginator) + require.NotNil(t, store.paginator(store.client, &s3.ListObjectsV2Input{ + Bucket: aws.String("xdrop"), + })) + }) +} + +func TestS3StorageEnsureBucket(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("head bucket success", func(t *testing.T) { + store := S3Storage{ + bucket: "xdrop", + client: fakeS3Client{ + headBucketFn: func(context.Context, *s3.HeadBucketInput, ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return &s3.HeadBucketOutput{}, nil + }, + }, + } + + require.NoError(t, store.EnsureBucket(ctx)) + }) + + t.Run("create bucket on missing bucket", func(t *testing.T) { + createCalls := 0 + store := S3Storage{ + bucket: "xdrop", + client: fakeS3Client{ + headBucketFn: func(context.Context, *s3.HeadBucketInput, ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, errors.New("missing") + }, + createBucketFn: func(context.Context, *s3.CreateBucketInput, ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + createCalls++ + return &s3.CreateBucketOutput{}, nil + }, + }, + } + + require.NoError(t, store.EnsureBucket(ctx)) + require.Equal(t, 1, createCalls) + }) + + t.Run("bucket already owned is ignored", func(t *testing.T) { + store := S3Storage{ + bucket: "xdrop", + client: fakeS3Client{ + headBucketFn: func(context.Context, *s3.HeadBucketInput, ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, errors.New("missing") + }, + createBucketFn: func(context.Context, *s3.CreateBucketInput, ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + return nil, errors.New("BucketAlreadyOwnedByYou") + }, + }, + } + + require.NoError(t, store.EnsureBucket(ctx)) + }) + + t.Run("create bucket failure", func(t *testing.T) { + store := S3Storage{ + bucket: "xdrop", + client: fakeS3Client{ + headBucketFn: func(context.Context, *s3.HeadBucketInput, ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + return nil, errors.New("missing") + }, + createBucketFn: func(context.Context, *s3.CreateBucketInput, ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + return nil, errors.New("boom") + }, + }, + } + + err := store.EnsureBucket(ctx) + require.ErrorContains(t, err, "create bucket") + }) +} + +func TestS3StoragePresignAndUploadOperations(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("presign upload success and error", func(t *testing.T) { + store := S3Storage{ + bucket: "xdrop", + presigner: fakePresigner{ + presignPutObjectFn: func(context.Context, *s3.PutObjectInput, ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return &v4.PresignedHTTPRequest{URL: "http://upload.example.test"}, nil + }, + }, + } + + url, err := store.PresignUpload(ctx, "object.bin", time.Minute) + require.NoError(t, err) + require.Equal(t, "http://upload.example.test", url) + + store.presigner = fakePresigner{ + presignPutObjectFn: func(context.Context, *s3.PutObjectInput, ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return nil, errors.New("boom") + }, + } + _, err = store.PresignUpload(ctx, "object.bin", time.Minute) + require.ErrorContains(t, err, "presign upload") + }) + + t.Run("presign download success and error", func(t *testing.T) { + store := S3Storage{ + bucket: "xdrop", + presigner: fakePresigner{ + presignGetObjectFn: func(context.Context, *s3.GetObjectInput, ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return &v4.PresignedHTTPRequest{URL: "http://download.example.test"}, nil + }, + }, + } + + url, err := store.PresignDownload(ctx, "object.bin", time.Minute) + require.NoError(t, err) + require.Equal(t, "http://download.example.test", url) + + store.presigner = fakePresigner{ + presignGetObjectFn: func(context.Context, *s3.GetObjectInput, ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return nil, errors.New("boom") + }, + } + _, err = store.PresignDownload(ctx, "object.bin", time.Minute) + require.ErrorContains(t, err, "presign download") + }) + + t.Run("put object success and error", func(t *testing.T) { + body := []byte("payload") + store := S3Storage{ + bucket: "xdrop", + uploader: fakeUploader{ + uploadFn: func(_ context.Context, input *s3.PutObjectInput, _ ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + uploaded, err := io.ReadAll(input.Body) + require.NoError(t, err) + require.Equal(t, body, uploaded) + require.Equal(t, "application/octet-stream", aws.ToString(input.ContentType)) + return &manager.UploadOutput{}, nil + }, + }, + } + + require.NoError(t, store.PutObject(ctx, "object.bin", body, "application/octet-stream")) + + store.uploader = fakeUploader{ + uploadFn: func(context.Context, *s3.PutObjectInput, ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return nil, errors.New("boom") + }, + } + err := store.PutObject(ctx, "object.bin", body, "application/octet-stream") + require.ErrorContains(t, err, "upload object") + }) +} + +func TestS3StorageDeletePrefix(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("list error", func(t *testing.T) { + store := S3Storage{ + bucket: "xdrop", + client: fakeS3Client{}, + paginator: func(s3.ListObjectsV2APIClient, *s3.ListObjectsV2Input) listObjectsV2Paginator { + return &fakePaginator{nextPageErr: errors.New("boom"), remaining: 1} + }, + } + + err := store.DeletePrefix(ctx, "transfers/demo/") + require.ErrorContains(t, err, "list objects") + }) + + t.Run("delete error", func(t *testing.T) { + store := S3Storage{ + bucket: "xdrop", + client: fakeS3Client{ + deleteObjectsFn: func(context.Context, *s3.DeleteObjectsInput, ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + return nil, errors.New("boom") + }, + }, + paginator: func(s3.ListObjectsV2APIClient, *s3.ListObjectsV2Input) listObjectsV2Paginator { + return &fakePaginator{ + pages: []*s3.ListObjectsV2Output{{ + Contents: []s3types.Object{{Key: aws.String("transfers/demo/one.bin")}}, + }}, + remaining: 1, + } + }, + } + + err := store.DeletePrefix(ctx, "transfers/demo/") + require.ErrorContains(t, err, "delete objects") + }) + + t.Run("success skips empty pages and nil keys", func(t *testing.T) { + deletedKeys := []string{} + store := S3Storage{ + bucket: "xdrop", + client: fakeS3Client{ + deleteObjectsFn: func(_ context.Context, input *s3.DeleteObjectsInput, _ ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + for _, object := range input.Delete.Objects { + deletedKeys = append(deletedKeys, aws.ToString(object.Key)) + } + return &s3.DeleteObjectsOutput{}, nil + }, + }, + paginator: func(s3.ListObjectsV2APIClient, *s3.ListObjectsV2Input) listObjectsV2Paginator { + return &fakePaginator{ + pages: []*s3.ListObjectsV2Output{ + {}, + { + Contents: []s3types.Object{ + {}, + {Key: aws.String("transfers/demo/one.bin")}, + {Key: aws.String("transfers/demo/two.bin")}, + }, + }, + }, + remaining: 2, + } + }, + } + + require.NoError(t, store.DeletePrefix(ctx, "transfers/demo/")) + require.Equal(t, []string{"transfers/demo/one.bin", "transfers/demo/two.bin"}, deletedKeys) + }) + + t.Run("skips delete calls when a page only has nil keys", func(t *testing.T) { + deleteCalls := 0 + store := S3Storage{ + bucket: "xdrop", + client: fakeS3Client{ + deleteObjectsFn: func(context.Context, *s3.DeleteObjectsInput, ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + deleteCalls++ + return &s3.DeleteObjectsOutput{}, nil + }, + }, + paginator: func(s3.ListObjectsV2APIClient, *s3.ListObjectsV2Input) listObjectsV2Paginator { + return &fakePaginator{ + pages: []*s3.ListObjectsV2Output{{ + Contents: []s3types.Object{{}}, + }}, + remaining: 1, + } + }, + } + + require.NoError(t, store.DeletePrefix(ctx, "transfers/demo/")) + require.Zero(t, deleteCalls) + }) +} + +type fakeS3Client struct { + createBucketFn func(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) + deleteObjectsFn func(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) + getObjectFn func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + headBucketFn func(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) + listObjectsFn func(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) +} + +func (c fakeS3Client) CreateBucket(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) { + if c.createBucketFn != nil { + return c.createBucketFn(ctx, params, optFns...) + } + return &s3.CreateBucketOutput{}, nil +} + +func (c fakeS3Client) DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + if c.deleteObjectsFn != nil { + return c.deleteObjectsFn(ctx, params, optFns...) + } + return &s3.DeleteObjectsOutput{}, nil +} + +func (c fakeS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + if c.getObjectFn != nil { + return c.getObjectFn(ctx, params, optFns...) + } + return &s3.GetObjectOutput{}, nil +} + +func (c fakeS3Client) HeadBucket(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options)) (*s3.HeadBucketOutput, error) { + if c.headBucketFn != nil { + return c.headBucketFn(ctx, params, optFns...) + } + return &s3.HeadBucketOutput{}, nil +} + +func (c fakeS3Client) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + if c.listObjectsFn != nil { + return c.listObjectsFn(ctx, params, optFns...) + } + return &s3.ListObjectsV2Output{}, nil +} + +type fakePresigner struct { + presignGetObjectFn func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) + presignPutObjectFn func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) +} + +func (p fakePresigner) PresignGetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + if p.presignGetObjectFn != nil { + return p.presignGetObjectFn(ctx, params, optFns...) + } + return &v4.PresignedHTTPRequest{}, nil +} + +func (p fakePresigner) PresignPutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + if p.presignPutObjectFn != nil { + return p.presignPutObjectFn(ctx, params, optFns...) + } + return &v4.PresignedHTTPRequest{}, nil +} + +type fakeUploader struct { + uploadFn func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) +} + +func (u fakeUploader) Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + if u.uploadFn != nil { + return u.uploadFn(ctx, input, opts...) + } + return &manager.UploadOutput{}, nil +} + +type fakePaginator struct { + nextPageErr error + pages []*s3.ListObjectsV2Output + remaining int +} + +func (p *fakePaginator) HasMorePages() bool { + return p.remaining > 0 +} + +func (p *fakePaginator) NextPage(context.Context, ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + if p.nextPageErr != nil { + return nil, p.nextPageErr + } + page := p.pages[0] + p.pages = p.pages[1:] + p.remaining-- + return page, nil +} + +type errReader struct{} + +func (errReader) Read([]byte) (int, error) { + return 0, errors.New("boom") +} + +func TestReadAllPropagatesReaderErrors(t *testing.T) { + t.Parallel() + + _, err := ReadAll(errReader{}) + require.ErrorContains(t, err, "boom") +} diff --git a/apps/web/.gitignore b/apps/web/.gitignore new file mode 100644 index 0000000..07ad785 --- /dev/null +++ b/apps/web/.gitignore @@ -0,0 +1,23 @@ +# Logs +logs +*.log +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/apps/web/eslint.config.js b/apps/web/eslint.config.js new file mode 100644 index 0000000..434e4a6 --- /dev/null +++ b/apps/web/eslint.config.js @@ -0,0 +1,29 @@ +import js from '@eslint/js' +import globals from 'globals' +import reactHooks from 'eslint-plugin-react-hooks' +import reactRefresh from 'eslint-plugin-react-refresh' +import tseslint from 'typescript-eslint' +import { defineConfig, globalIgnores } from 'eslint/config' + +export default defineConfig([ + globalIgnores(['coverage', 'dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + js.configs.recommended, + tseslint.configs.recommended, + reactHooks.configs.flat.recommended, + reactRefresh.configs.vite, + ], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, + }, + { + files: ['src/features/upload/TransferContext.tsx', 'src/features/upload/UploadStudio.tsx'], + rules: { + 'react-refresh/only-export-components': 'off', + }, + }, +]) diff --git a/apps/web/index.html b/apps/web/index.html new file mode 100644 index 0000000..6ede3dd --- /dev/null +++ b/apps/web/index.html @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + Open Source Encrypted File Transfer in the Browser | Xdrop + + +
+ + + diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 0000000..2a7b77c --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,49 @@ +{ + "name": "@xdrop/web", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build && bun ./scripts/generate-seo-assets.mjs", + "lint": "eslint .", + "preview": "vite preview", + "test": "vitest run", + "test:coverage": "vitest run --coverage", + "test:watch": "vitest", + "typecheck": "tsc --noEmit -p tsconfig.app.json" + }, + "dependencies": { + "@fontsource-variable/instrument-sans": "^5.2.8", + "@fontsource/fraunces": "^5.2.9", + "@xdrop/shared": "workspace:*", + "@zip.js/zip.js": "^2.8.10", + "idb": "^8.0.3", + "qrcode": "^1.5.4", + "react": "^19.2.4", + "react-dom": "^19.2.4", + "react-router-dom": "^7.9.6", + "zod": "^4.1.12" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "@testing-library/jest-dom": "^6.9.1", + "@testing-library/react": "^16.3.0", + "@testing-library/user-event": "^14.6.1", + "@types/node": "^25.5.0", + "@types/qrcode": "^1.5.5", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitest/coverage-v8": "^4.0.7", + "@vitejs/plugin-react": "^6.0.1", + "eslint": "^10.1.0", + "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-refresh": "^0.5.2", + "fake-indexeddb": "^6.2.2", + "globals": "^17.4.0", + "jsdom": "^29.0.1", + "typescript": "~5.9.3", + "typescript-eslint": "^8.56.1", + "vite": "^8.0.1", + "vitest": "^4.0.7" + } +} diff --git a/apps/web/public/apple-touch-icon.png b/apps/web/public/apple-touch-icon.png new file mode 100644 index 0000000..c9ee047 Binary files /dev/null and b/apps/web/public/apple-touch-icon.png differ diff --git a/apps/web/public/brand-lockup-horizontal.png b/apps/web/public/brand-lockup-horizontal.png new file mode 100644 index 0000000..711cee2 Binary files /dev/null and b/apps/web/public/brand-lockup-horizontal.png differ diff --git a/apps/web/public/brand-lockup-horizontal.svg b/apps/web/public/brand-lockup-horizontal.svg new file mode 100644 index 0000000..c0c8262 --- /dev/null +++ b/apps/web/public/brand-lockup-horizontal.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Xdrop + + + diff --git a/apps/web/public/brand-symbol-192.png b/apps/web/public/brand-symbol-192.png new file mode 100644 index 0000000..27ad98d Binary files /dev/null and b/apps/web/public/brand-symbol-192.png differ diff --git a/apps/web/public/brand-symbol-512.png b/apps/web/public/brand-symbol-512.png new file mode 100644 index 0000000..8fffae1 Binary files /dev/null and b/apps/web/public/brand-symbol-512.png differ diff --git a/apps/web/public/brand-symbol-maskable-512.png b/apps/web/public/brand-symbol-maskable-512.png new file mode 100644 index 0000000..8f441c2 Binary files /dev/null and b/apps/web/public/brand-symbol-maskable-512.png differ diff --git a/apps/web/public/brand-symbol-maskable.svg b/apps/web/public/brand-symbol-maskable.svg new file mode 100644 index 0000000..d15bf10 --- /dev/null +++ b/apps/web/public/brand-symbol-maskable.svg @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/web/public/brand-symbol.svg b/apps/web/public/brand-symbol.svg new file mode 100644 index 0000000..737db9a --- /dev/null +++ b/apps/web/public/brand-symbol.svg @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/web/public/favicon.svg b/apps/web/public/favicon.svg new file mode 100644 index 0000000..0bc249e --- /dev/null +++ b/apps/web/public/favicon.svg @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/web/public/icons.svg b/apps/web/public/icons.svg new file mode 100644 index 0000000..4367741 --- /dev/null +++ b/apps/web/public/icons.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/web/public/manifest.webmanifest b/apps/web/public/manifest.webmanifest new file mode 100644 index 0000000..fc66d16 --- /dev/null +++ b/apps/web/public/manifest.webmanifest @@ -0,0 +1,35 @@ +{ + "name": "Xdrop", + "short_name": "Xdrop", + "description": "Xdrop is an open source file transfer app that encrypts files in your browser and keeps plaintext file names, contents, and keys off the server.", + "theme_color": "#12140f", + "background_color": "#f7f2e8", + "display": "standalone", + "start_url": "/", + "icons": [ + { + "src": "/brand-symbol.svg", + "sizes": "any", + "type": "image/svg+xml", + "purpose": "any" + }, + { + "src": "/brand-symbol-192.png", + "sizes": "192x192", + "type": "image/png", + "purpose": "any" + }, + { + "src": "/brand-symbol-512.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "any" + }, + { + "src": "/brand-symbol-maskable-512.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "maskable" + } + ] +} diff --git a/apps/web/public/sw.js b/apps/web/public/sw.js new file mode 100644 index 0000000..66f90b1 --- /dev/null +++ b/apps/web/public/sw.js @@ -0,0 +1,83 @@ +const CACHE_NAME = 'xdrop-static-v4' +const STATIC_ASSETS = [ + '/manifest.webmanifest', + '/brand-symbol.svg', + '/favicon.svg', + '/brand-symbol-maskable.svg', + '/brand-symbol-192.png', + '/brand-symbol-512.png', + '/brand-symbol-maskable-512.png', + '/brand-lockup-horizontal.svg', + '/brand-lockup-horizontal.png', + '/apple-touch-icon.png', + '/icons.svg', +] + +self.addEventListener('install', (event) => { + event.waitUntil( + caches + .open(CACHE_NAME) + .then((cache) => cache.addAll(STATIC_ASSETS)) + .then(() => self.skipWaiting()), + ) +}) + +self.addEventListener('activate', (event) => { + event.waitUntil( + caches + .keys() + .then((keys) => + Promise.all(keys.filter((key) => key !== CACHE_NAME).map((key) => caches.delete(key))), + ) + .then(() => self.clients.claim()), + ) +}) + +self.addEventListener('fetch', (event) => { + const request = event.request + if (request.method !== 'GET') { + return + } + + const url = new URL(request.url) + if ( + url.origin !== self.location.origin || + url.pathname.startsWith('/api/') || + url.pathname.startsWith('/xdrop/') + ) { + return + } + + if (request.mode === 'navigate') { + event.respondWith(fetch(request)) + return + } + + if (!shouldCache(url.pathname)) { + return + } + + event.respondWith( + caches.match(request).then((cached) => { + if (cached) { + return cached + } + + return fetch(request) + .then((response) => { + if (!response.ok || response.type === 'opaque') { + return response + } + + const copy = response.clone() + void caches.open(CACHE_NAME).then((cache) => cache.put(request, copy)) + return response + }) + .catch(() => cached) + }), + ) +}) + +function shouldCache(pathname) { + return pathname.startsWith('/assets/') || STATIC_ASSETS.includes(pathname) +} diff --git a/apps/web/scripts/generate-seo-assets.mjs b/apps/web/scripts/generate-seo-assets.mjs new file mode 100644 index 0000000..a1bb550 --- /dev/null +++ b/apps/web/scripts/generate-seo-assets.mjs @@ -0,0 +1,278 @@ +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { JSDOM } from 'jsdom' + +const currentDirectory = dirname(fileURLToPath(import.meta.url)) +const distDirectory = join(currentDirectory, '..', 'dist') +const siteUrl = process.env.VITE_SITE_URL?.trim().replace(/\/+$/u, '') || '' + +const SITE_NAME = 'Xdrop' +const DEFAULT_OG_IMAGE_PATH = '/brand-lockup-horizontal.png' +const DEFAULT_LOGO_PATH = '/brand-symbol-512.png' +const DEFAULT_OG_IMAGE_ALT = 'Xdrop horizontal brand lockup' +const DEFAULT_OG_TYPE = 'website' +const REPOSITORY_URL = 'https://github.com/xixu-me/xdrop' +const DEFAULT_ROBOTS = + 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' +const PRIVATE_ROBOTS = 'noindex, nofollow, noarchive, nosnippet' +const STRUCTURED_DATA_ID = 'xdrop-structured-data-static' + +const HOME_TITLE = 'Open Source Encrypted File Transfer in the Browser | Xdrop' +const HOME_DESCRIPTION = + 'Xdrop is an open source file transfer app that encrypts files in your browser and keeps plaintext file names, contents, and keys off the server.' + +const routeShells = [ + { + outputPath: 'index.html', + pagePath: '/', + title: HOME_TITLE, + description: HOME_DESCRIPTION, + robots: DEFAULT_ROBOTS, + structuredData: getHomeStructuredData(), + }, + { + outputPath: 'transfers/index.html', + pagePath: '/transfers', + title: 'Manage Transfers on This Device | Xdrop', + description: + 'Manage encrypted transfers stored in this browser. There is no account or cross-device history.', + robots: PRIVATE_ROBOTS, + exposeUrl: false, + }, + { + outputPath: 'share/index.html', + pagePath: '/share/', + title: 'Share the Full Link | Xdrop', + description: + 'Review upload status and copy the full share link for a browser-encrypted transfer.', + robots: PRIVATE_ROBOTS, + exposeUrl: false, + }, + { + outputPath: 't/index.html', + pagePath: '/t/', + title: 'Download and Decrypt in the Browser | Xdrop', + description: + 'Download files from this transfer and decrypt them in the browser. The decryption key stays in the share link fragment.', + robots: PRIVATE_ROBOTS, + exposeUrl: false, + }, + { + outputPath: 'not-found/index.html', + title: 'Page Not Found | Xdrop', + description: + 'The address does not map to a page in Xdrop. If this came from a shared transfer, ask for the full URL, including the #k=... decryption fragment.', + robots: PRIVATE_ROBOTS, + exposeUrl: false, + }, +] + +const robotsLines = [ + 'User-agent: *', + 'Allow: /', + 'Disallow: /share/', + 'Disallow: /t/', + 'Disallow: /transfers', +] + +if (siteUrl) { + robotsLines.push(`Sitemap: ${siteUrl}/sitemap.xml`) +} + +await writeFile(join(distDirectory, 'robots.txt'), `${robotsLines.join('\n')}\n`, 'utf8') + +if (!siteUrl) { + await rm(join(distDirectory, 'sitemap.xml'), { force: true }) + console.warn('[seo] Skipped sitemap.xml generation because VITE_SITE_URL is not set.') +} + +const sitemap = siteUrl + ? ` + + + ${siteUrl}/ + weekly + 1.0 + + +` + : '' + +if (sitemap) { + await writeFile(join(distDirectory, 'sitemap.xml'), sitemap, 'utf8') +} + +const template = await readFile(join(distDirectory, 'index.html'), 'utf8') + +for (const shell of routeShells) { + const outputFile = join(distDirectory, shell.outputPath) + await mkdir(dirname(outputFile), { recursive: true }) + await writeFile(outputFile, renderRouteShell(template, shell), 'utf8') +} + +console.info( + `[seo] Generated route HTML shells${siteUrl ? `, robots.txt, and sitemap.xml for ${siteUrl}` : ' and robots.txt'}.`, +) + +function renderRouteShell(template, shell) { + const dom = new JSDOM(template) + const { document } = dom.window + const canonicalUrl = + shell.exposeUrl === false || !shell.pagePath ? undefined : toAbsoluteUrl(shell.pagePath) + const imageUrl = toAbsoluteUrl(DEFAULT_OG_IMAGE_PATH) + + document.title = shell.title + setMetaByName(document, 'description', shell.description) + setMetaByName(document, 'robots', shell.robots) + setMetaByName(document, 'application-name', SITE_NAME) + setMetaByName(document, 'apple-mobile-web-app-title', SITE_NAME) + setMetaByName(document, 'twitter:card', 'summary_large_image') + setMetaByName(document, 'twitter:title', shell.title) + setMetaByName(document, 'twitter:description', shell.description) + setMetaByName(document, 'twitter:image', imageUrl) + + setMetaByProperty(document, 'og:site_name', SITE_NAME) + setMetaByProperty(document, 'og:type', DEFAULT_OG_TYPE) + setMetaByProperty(document, 'og:title', shell.title) + setMetaByProperty(document, 'og:description', shell.description) + setMetaByProperty(document, 'og:image', imageUrl) + setMetaByProperty(document, 'og:image:alt', DEFAULT_OG_IMAGE_ALT) + + if (canonicalUrl) { + setMetaByProperty(document, 'og:url', canonicalUrl) + ensureCanonicalLink(document, canonicalUrl) + } else { + removeCanonicalLink(document) + removeMetaByProperty(document, 'og:url') + } + + syncStructuredData(document, shell.structuredData) + + return `\n${document.documentElement.outerHTML}\n` +} + +function setMetaByName(document, name, content) { + let meta = document.head.querySelector(`meta[name="${name}"]`) + + if (!meta) { + meta = document.createElement('meta') + meta.setAttribute('name', name) + document.head.append(meta) + } + + meta.setAttribute('content', content) +} + +function setMetaByProperty(document, property, content) { + let meta = document.head.querySelector(`meta[property="${property}"]`) + + if (!meta) { + meta = document.createElement('meta') + meta.setAttribute('property', property) + document.head.append(meta) + } + + meta.setAttribute('content', content) +} + +function removeMetaByProperty(document, property) { + document.head.querySelector(`meta[property="${property}"]`)?.remove() +} + +function ensureCanonicalLink(document, href) { + let link = document.head.querySelector('link[rel="canonical"]') + + if (!link) { + link = document.createElement('link') + link.setAttribute('rel', 'canonical') + document.head.append(link) + } + + link.setAttribute('href', href) +} + +function removeCanonicalLink(document) { + document.head.querySelector('link[rel="canonical"]')?.remove() +} + +function syncStructuredData(document, structuredData) { + const existing = document.getElementById(STRUCTURED_DATA_ID) + + if (!structuredData) { + existing?.remove() + return + } + + const script = existing ?? document.createElement('script') + script.id = STRUCTURED_DATA_ID + script.setAttribute('type', 'application/ld+json') + script.textContent = JSON.stringify(structuredData) + + if (!existing) { + document.head.append(script) + } +} + +function getHomeStructuredData() { + const homeUrl = toAbsoluteUrl('/') + const organizationId = `${homeUrl}#organization` + const logoUrl = toAbsoluteUrl(DEFAULT_LOGO_PATH) + const imageUrl = toAbsoluteUrl(DEFAULT_OG_IMAGE_PATH) + + return [ + { + '@context': 'https://schema.org', + '@type': 'WebSite', + name: SITE_NAME, + url: homeUrl, + description: HOME_DESCRIPTION, + image: imageUrl, + publisher: { + '@id': organizationId, + }, + }, + { + '@context': 'https://schema.org', + '@type': 'SoftwareApplication', + name: SITE_NAME, + applicationCategory: 'UtilitiesApplication', + operatingSystem: 'Any', + offers: { + '@type': 'Offer', + price: '0', + priceCurrency: 'USD', + }, + description: HOME_DESCRIPTION, + url: homeUrl, + image: imageUrl, + publisher: { + '@id': organizationId, + }, + sameAs: [REPOSITORY_URL], + }, + { + '@context': 'https://schema.org', + '@id': organizationId, + '@type': 'Organization', + name: SITE_NAME, + url: homeUrl, + sameAs: [REPOSITORY_URL], + image: imageUrl, + logo: { + '@type': 'ImageObject', + url: logoUrl, + width: 512, + height: 512, + }, + }, + ] +} + +function toAbsoluteUrl(path) { + if (!siteUrl) { + return path + } + + return new URL(path, `${siteUrl}/`).toString() +} diff --git a/apps/web/src/app/App.tsx b/apps/web/src/app/App.tsx new file mode 100644 index 0000000..9184217 --- /dev/null +++ b/apps/web/src/app/App.tsx @@ -0,0 +1,83 @@ +/** + * Root router for the browser application. + */ + +import { createBrowserRouter, RouterProvider } from 'react-router-dom' + +import { PageStateCard } from '@/components/ui/PageStateCard' +import { Shell } from './Shell' + +function RouteHydrateFallback() { + return ( + + ) +} + +function lazyRoute(load: () => Promise) { + return { + lazy: load, + hydrateFallbackElement: , + } +} + +/** The router keeps navigation declarative and co-locates each page with its route. */ +const router = createBrowserRouter([ + { + path: '/', + Component: Shell, + children: [ + { + ...lazyRoute(async () => { + const { SenderRouteLayout } = await import('./routes/SenderRouteLayout') + return { Component: SenderRouteLayout } + }), + children: [ + { + index: true, + ...lazyRoute(async () => { + const { HomePage } = await import('./routes/HomePage') + return { Component: HomePage } + }), + }, + { + path: 'share/:transferId', + ...lazyRoute(async () => { + const { SharePage } = await import('./routes/SharePage') + return { Component: SharePage } + }), + }, + { + path: 'transfers', + ...lazyRoute(async () => { + const { HistoryPage } = await import('./routes/HistoryPage') + return { Component: HistoryPage } + }), + }, + ], + }, + { + path: 't/:transferId', + ...lazyRoute(async () => { + const { ReceivePage } = await import('./routes/ReceivePage') + return { Component: ReceivePage } + }), + }, + { + path: '*', + ...lazyRoute(async () => { + const { NotFoundPage } = await import('./routes/NotFoundPage') + return { Component: NotFoundPage } + }), + }, + ], + }, +]) + +/** App renders the router provider used by the rest of the browser UI. */ +export function App() { + return +} diff --git a/apps/web/src/app/Shell.test.tsx b/apps/web/src/app/Shell.test.tsx new file mode 100644 index 0000000..3012787 --- /dev/null +++ b/apps/web/src/app/Shell.test.tsx @@ -0,0 +1,38 @@ +import { render, screen } from '@testing-library/react' +import { MemoryRouter, Route, Routes } from 'react-router-dom' +import { describe, expect, it } from 'vitest' + +import { AUTHOR_NAME, AUTHOR_URL, LICENSE_URL, REPOSITORY_URL } from '@/lib/seo/site' + +import { Shell } from './Shell' + +describe('Shell', () => { + it('renders navigation and the nested outlet', () => { + render( + + + }> + Transfers content} /> + + + , + ) + + expect(screen.getByRole('link', { name: 'Xdrop' })).toHaveAttribute('href', '/') + expect(screen.getByRole('link', { name: 'Send' })).toHaveAttribute('href', '/') + expect(screen.getByRole('link', { name: 'Transfers' })).toHaveAttribute('href', '/transfers') + expect(screen.getByText('Transfers content')).toBeInTheDocument() + expect(screen.getByRole('link', { name: 'View the license' }).closest('p')).toHaveTextContent( + `Developed by ${AUTHOR_NAME} and released under the GNU Affero General Public License v3.0 only.`, + ) + expect(screen.getByRole('link', { name: 'View the license' }).closest('p')).toHaveTextContent( + 'more information and source code on GitHub.', + ) + expect(screen.getByRole('link', { name: 'View the license' })).toHaveAttribute( + 'href', + LICENSE_URL, + ) + expect(screen.getByRole('link', { name: 'GitHub' })).toHaveAttribute('href', REPOSITORY_URL) + expect(screen.getByRole('link', { name: AUTHOR_NAME })).toHaveAttribute('href', AUTHOR_URL) + }) +}) diff --git a/apps/web/src/app/Shell.tsx b/apps/web/src/app/Shell.tsx new file mode 100644 index 0000000..e303949 --- /dev/null +++ b/apps/web/src/app/Shell.tsx @@ -0,0 +1,55 @@ +import { NavLink, Outlet } from 'react-router-dom' + +import { AUTHOR_NAME, AUTHOR_URL, LICENSE_URL, REPOSITORY_URL } from '@/lib/seo/site' + +/** Shell provides the shared navigation and decorative chrome around route content. */ +export function Shell() { + return ( +
+