From f5b68fb28af9b3352de9ac5ffb900587b894814b Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Mon, 10 Aug 2026 17:11:04 -0500 Subject: [PATCH] chore(hooks): install working pre-commit/pre-push git hooks .git/hooks/pre-commit was a pre-commit-framework shim on repos branched from main with no .pre-commit-config.yaml, so every commit died with InvalidConfigError. v2.1.x already had a config but it predated the framework's pre-commit/pre-push stage naming, hand-rolled several checks with silent `|| true` tool-missing fallbacks, and linted Python with flake8/black/isort. - Modernize .pre-commit-config.yaml: pre-commit/pre-push stages, add missing hygiene hooks, framework-managed gitleaks/shellcheck/golangci-lint (pinned v1.64.8 to match CI, matching go-client-release.yml) instead of system-PATH-dependent local hooks, actionlint (shellcheck sub-check disabled -- redundant with the dedicated shellcheck hook), bandit + Dockerfile rootless check at pre-push. - Migrate Python lint from flake8/black/isort to ruff (backend-python.md): canonical [tool.ruff] in pyproject.toml, blocking hook scoped to --select=F,E9,B (flake8's prior bar), full canonical rule set advisory (~1,700 pre-existing findings, mostly missing docstrings/pyupgrade across 5 services -- not in scope here, tracked via the pyproject.toml comment for incremental adoption). Fixed all 50 findings in the blocking scope, including a real bug (dpop_service.py referenced `traceback` without importing it) and 4 missing `raise ... from`. Removed flake8/black/isort from requirements-dev.txt, .flake8, and CI. Also fixed pre-existing bandit findings (B104 false-positives on server listen sockets/config defaults, annotated; B113 missing requests timeout) and shellcheck findings (SC2145/SC2155/SC2034) surfaced by wiring the tools in for the first time. - Copy scripts/install-pre-commit.sh + lib/detect-os.sh + hooks/check-dockerfile-rootless.sh from admin; fix a worktree bug in --verify (used "$root/.git/hooks" instead of --git-common-dir, so it's a file not a dir inside a worktree and always reports NOT INSTALLED). - Add install-hooks/verify-hooks Makefile targets; make setup depend on install-hooks; update lint/fix-lint/format for ruff. - Update docs (CONTRIBUTING/DEVELOPMENT/PRE_COMMIT/STANDARDS/WORKFLOWS) to match. Verified: pre-commit run --all-files and --hook-stage pre-push both pass clean. make lint and make install-hooks/verify-hooks pass. Unit tests for touched files pass where the local environment has their deps installed; 6 pre-existing failures (dpop_service crypto, observability, saml/scim missing `saml2` package) reproduce identically against the unmodified files and are out of scope here. Co-Authored-By: Claude Opus 5 --- .claude/.claude/go-backend.md | 1 - .cm/gitstream.cm | 4 +- .env-example | 4 +- .env.license.example | 2 +- .flake8 | 25 -- .github/scripts/extract-release-notes.sh | 10 +- .github/workflows/gitstream.yml | 2 +- .github/workflows/server-release.yml | 4 +- .pre-commit-config.yaml | 186 ++++++++----- LICENSE.md | 2 - Makefile | 67 ++--- README.md | 18 +- dhcp-server/requirements-dev.txt | 5 +- dns-server/TESTING.md | 12 +- dns-server/app/grpc_server.py | 2 +- dns-server/app/observability.py | 1 - dns-server/app/services/cert_manager.py | 2 +- dns-server/app/services/rate_limiter.py | 2 +- dns-server/app/utils/jwt_verify.py | 7 +- dns-server/apt-packages.txt | 2 +- dns-server/bandit-report.json | 2 +- dns-server/docker-compose.yml | 2 +- dns-server/requirements-dev.txt | 5 +- dns-server/tests/README.md | 6 +- dns-server/tests/test_ioc_blocking.py | 72 ++--- dns-server/tests_full_future/__init__.py | 2 +- dns-server/tests_full_future/conftest.py | 10 +- .../test_client_config_api.py | 186 ++++++------- .../test_client_config_tenant.py | 4 +- .../tests_full_future/test_ioc_manager.py | 260 +++++++++--------- .../test_prometheus_metrics.py | 214 +++++++------- .../test_selective_dns_routing.py | 214 +++++++------- .../tests_full_future/test_whois_manager.py | 134 ++++----- dns-server/tests_full_future/unittests.py | 2 +- docker-compose.license.yml | 2 +- docker-compose.yml | 2 +- docs/.gitignore | 2 +- docs/API.md | 122 ++++---- docs/ARCHITECTURE.md | 44 +-- docs/CONTRIBUTING.md | 64 ++--- docs/DEVELOPMENT.md | 117 +++----- docs/PRE_COMMIT.md | 11 +- docs/RELEASE_NOTES.md | 2 +- docs/STANDARDS.md | 2 +- docs/TOKEN_MANAGEMENT.md | 4 +- docs/USAGE.md | 26 +- docs/WORKFLOWS.md | 9 +- docs/mkdocs.yml | 6 +- docs/stylesheets/extra.css | 6 +- entrypoint.yml | 2 +- install.py | 160 +++++------ manager/backend/app/blueprints/analytics.py | 1 - manager/backend/app/blueprints/dhcp.py | 1 - .../backend/app/blueprints/machine_clients.py | 5 +- manager/backend/app/blueprints/mfa.py | 1 - .../app/blueprints/oidc_trust_anchors.py | 2 +- manager/backend/app/blueprints/saml.py | 1 - manager/backend/app/blueprints/scim.py | 23 +- manager/backend/app/models/dhcp.py | 2 +- manager/backend/app/observability.py | 1 - manager/backend/app/schema.py | 2 +- .../backend/app/services/config_service.py | 2 - manager/backend/app/services/dpop_service.py | 6 +- .../app/services/ioc_ingestion_service.py | 1 - manager/backend/app/services/saml_service.py | 5 +- .../backend/app/services/signing_provider.py | 2 +- manager/backend/app/services/sso_service.py | 6 +- manager/backend/app/services/whois_service.py | 2 - manager/backend/app/utils/crypto.py | 2 +- manager/backend/app/utils/decorators.py | 2 +- manager/backend/requirements-dev.txt | 5 +- manager/tests/test_manager_api.py | 58 ++-- ntp-server/bins/server.py | 4 +- ntp-server/requirements-dev.txt | 5 +- pyproject.toml | 66 ++--- scripts/deploy-alpha.sh | 6 +- scripts/deploy-beta.sh | 8 +- scripts/hooks/check-dockerfile-rootless.sh | 67 +++++ scripts/init-postgres.sql | 6 +- scripts/install-pre-commit.sh | 124 +++++++++ scripts/lib/detect-os.sh | 178 ++++++++++++ scripts/test-integration.sh | 2 +- squawk-client-go/Makefile | 24 +- squawk-client-go/README-License.md | 2 +- squawk-client-go/README.md | 6 +- squawk-client-go/cmd/squawk-client/main.go | 14 +- squawk-client-go/pkg/client/doh_client.go | 46 ++-- squawk-client-go/pkg/config/config.go | 16 +- squawk-client-go/pkg/forwarder/forwarder.go | 2 +- squawk-client-go/pkg/license/validator.go | 10 +- squawk-client-go/pkg/logger/logger.go | 2 +- .../pkg/performance/dns_performance.go | 176 ++++++------ squawk-client/Dockerfile | 2 +- squawk-client/bins/client.py | 12 +- squawk-client/bins/k8s-client.py | 2 +- squawk-client/bins/systray.py | 2 +- squawk-client/docker-compose.yml | 8 +- squawk-client/requirements-dev.txt | 4 +- squawk-client/tests/test_client.py | 54 ++-- .../web/apps/_default/static/index.html | 24 +- .../web/apps/_minimal/static/README.md | 1 - .../web/apps/_scaffold/static/README.md | 1 - .../web/apps/_scaffold/static/js/utils.js | 16 +- .../web/apps/_scaffold/templates/README.md | 1 - .../web/apps/_scaffold/templates/index.html | 2 +- tests/load-test.js | 60 ++-- tests/smoke/beta/run_beta.sh | 1 - tests/test_cache.py | 80 +++--- tests/test_installer.py | 94 +++---- website/.eslintrc.json | 2 +- website/.gitignore | 2 +- website/CLOUDFLARE_CONFIG.md | 2 +- website/README.md | 8 +- website/components/Layout.js | 26 +- website/docs/CLOUDFLARE_CONFIG.md | 2 +- website/docs/README.md | 8 +- website/next.config.js | 16 +- website/pages/404.js | 4 +- website/pages/500.js | 4 +- website/pages/_app.js | 2 +- website/pages/contact.js | 14 +- website/pages/documentation.js | 22 +- website/pages/download.js | 46 ++-- website/pages/enterprise.js | 32 +-- website/pages/features.js | 20 +- website/pages/index.js | 60 ++-- website/pages/pricing.js | 8 +- website/public/_headers | 2 +- website/public/_redirects | 2 +- website/public/css/style.css | 14 +- website/public/js/main.js | 56 ++-- website/public/test.html | 2 +- website/styles/globals.css | 4 +- 133 files changed, 1983 insertions(+), 1685 deletions(-) delete mode 100644 .flake8 create mode 100755 scripts/hooks/check-dockerfile-rootless.sh create mode 100755 scripts/install-pre-commit.sh create mode 100755 scripts/lib/detect-os.sh diff --git a/.claude/.claude/go-backend.md b/.claude/.claude/go-backend.md index 31034728..8a741171 100644 --- a/.claude/.claude/go-backend.md +++ b/.claude/.claude/go-backend.md @@ -197,4 +197,3 @@ CMD ["/app"] - Integration tests: Container interactions - Smoke tests: Build, run, health checks, API endpoints - Performance tests: Throughput, latency benchmarks - diff --git a/.cm/gitstream.cm b/.cm/gitstream.cm index b915d589..0dddc323 100644 --- a/.cm/gitstream.cm +++ b/.cm/gitstream.cm @@ -13,7 +13,7 @@ automations: label: "{{ calc.etr }} min review" color: {{ 'E94637' if (calc.etr >= 20) else ('FBBD10' if (calc.etr >= 5) else '36A853') }} code_experts: - if: + if: - true run: - action: add-comment@v1 @@ -22,4 +22,4 @@ automations: {{ repo | explainCodeExperts(gt=10) }} calc: - etr: {{ branch | estimatedReviewTime }} \ No newline at end of file + etr: {{ branch | estimatedReviewTime }} diff --git a/.env-example b/.env-example index 4c312cc8..e125ea11 100644 --- a/.env-example +++ b/.env-example @@ -59,7 +59,7 @@ USER_REGISTRATION_REQUIRES_VERIFICATION=true USER_REGISTRATION_REQUIRES_APPROVAL=false BLOCK_PREVIOUS_PASSWORD_NUM=5 -# Session Management +# Session Management SESSION_TIMEOUT=3600 SESSION_SECRET=change-this-secret-key-in-production MFA_SESSION_TIMEOUT=28800 @@ -135,4 +135,4 @@ NETWORK_MODE=bridge # External Services UPSTREAM_DNS=8.8.8.8,1.1.1.1 DNS_OVER_TLS=false -DNS_OVER_TLS_HOSTNAME= \ No newline at end of file +DNS_OVER_TLS_HOSTNAME= diff --git a/.env.license.example b/.env.license.example index 7a63db70..d0c7b0bf 100644 --- a/.env.license.example +++ b/.env.license.example @@ -23,4 +23,4 @@ ADMIN_EMAIL=admin@squawkdns.com # Domain Configuration DNS_DOMAIN=dns.squawkdns.com CONSOLE_DOMAIN=console.squawkdns.com -LICENSE_DOMAIN=license.squawkdns.com \ No newline at end of file +LICENSE_DOMAIN=license.squawkdns.com diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 872c28c5..00000000 --- a/.flake8 +++ /dev/null @@ -1,25 +0,0 @@ -[flake8] -# Critical errors (E9xx, F6xx, F7xx, F8xx) + unused imports (F401) + bugbear -# (B, requires flake8-bugbear). E501 stays off: several existing files run -# well past 120 cols and reformatting them is a separate, larger cleanup -# (see pyproject.toml [tool.black] / [tool.isort] for the advisory formatters -# that do enforce 120 cols, run via `make lint` but non-blocking). -# Warnings reported but not blocking (set in Makefile) -max-line-length = 120 -extend-ignore = E203, E266, E501, W503 -exclude = - .git, - __pycache__, - .venv, - venv, - */venv, - node_modules, - .pytest_cache, - htmlcov, - .mypy_cache, - alembic, - migrations, - */alembic, - */migrations -# Select critical error codes + unused imports + bugbear-class checks -select = E9,F63,F7,F821,F401,B diff --git a/.github/scripts/extract-release-notes.sh b/.github/scripts/extract-release-notes.sh index 2cd979c3..01c6431a 100755 --- a/.github/scripts/extract-release-notes.sh +++ b/.github/scripts/extract-release-notes.sh @@ -26,11 +26,11 @@ EOF # Check if release notes file exists if [ -f "$RELEASE_NOTES_FILE" ]; then echo "Found release notes file: $RELEASE_NOTES_FILE" - + # Extract the first 400 lines to avoid hitting GitHub's limit echo "Extracting content from release notes..." head -n 400 "$RELEASE_NOTES_FILE" >> "$OUTPUT_FILE" - + # Add footer cat >> "$OUTPUT_FILE" << EOF @@ -83,12 +83,12 @@ docker run -p 8080:8080 \\ \`\`\` EOF fi - + echo "Successfully extracted $(wc -l < "$OUTPUT_FILE") lines to $OUTPUT_FILE" else echo "Release notes file not found: $RELEASE_NOTES_FILE" echo "Using minimal release body..." - + cat >> "$OUTPUT_FILE" << EOF High-performance DNS-over-HTTPS ${COMPONENT} with comprehensive security features. @@ -113,4 +113,4 @@ cat >> "$OUTPUT_FILE" << EOF EOF -echo "Release body created: $OUTPUT_FILE ($(wc -l < "$OUTPUT_FILE") lines)" \ No newline at end of file +echo "Release body created: $OUTPUT_FILE ($(wc -l < "$OUTPUT_FILE") lines)" diff --git a/.github/workflows/gitstream.yml b/.github/workflows/gitstream.yml index 0f898388..70dc08b8 100644 --- a/.github/workflows/gitstream.yml +++ b/.github/workflows/gitstream.yml @@ -15,7 +15,7 @@ on: description: the head sha required: true base_ref: - description: the base ref + description: the base ref required: true installation_id: description: the installation id diff --git a/.github/workflows/server-release.yml b/.github/workflows/server-release.yml index a477ff10..8d7fda0c 100644 --- a/.github/workflows/server-release.yml +++ b/.github/workflows/server-release.yml @@ -64,8 +64,8 @@ jobs: - name: Run linting on complete application image run: | - docker run --rm -w /app/dns-server squawk-dns-server:release-test python3.13 -m flake8 app/ --count --select=E9,F63,F7,F82 --show-source --statistics || true - docker run --rm -w /app/dns-server squawk-dns-server:release-test python3.13 -m flake8 app/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics || true + docker run --rm -w /app/dns-server squawk-dns-server:release-test python3.13 -m ruff check app/ --select=E9,F,B --statistics || true + docker run --rm -w /app/dns-server squawk-dns-server:release-test python3.13 -m ruff check app/ --exit-zero --statistics || true # Build and release job only runs when version tag created build-platform: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ce2c4383..cb5940cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,31 +1,40 @@ -default_stages: [commit] +# Squawk git hooks. +# +# Install: make install-hooks +# Verify: make verify-hooks +# Run all: pre-commit run --all-files +# +# Two stages, matching devops.md "Git Hooks (Mandatory)": +# pre-commit — fast checks only (<30s), blocks the commit +# pre-push — heavier security scans, blocks the push +# +# Revs are pinned. Update deliberately via `pre-commit autoupdate`, never by +# floating to a branch. +default_install_hook_types: [pre-commit, pre-push] +default_stages: [pre-commit] fail_fast: false repos: - # Secrets detection - - repo: https://github.com/gitleaks/gitleaks - rev: v8.18.0 - hooks: - - id: gitleaks - name: Detect secrets with gitleaks - entry: gitleaks protect --staged --source . - language: system - pass_filenames: false - stages: [commit, push] - - # End-of-file fixer + # ── Hygiene ──────────────────────────────────────────────────────────────── - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer name: Fix end of file - id: trailing-whitespace name: Fix trailing whitespace + - id: check-merge-conflict + name: Check merge conflict + - id: check-case-conflict + name: Check case conflict + - id: check-added-large-files + name: Check for added large files + args: [--maxkb=1024] - id: check-yaml name: Check YAML syntax - args: [--unsafe] - # Helm chart templates are Go-template text, not parseable YAML - # until rendered (use `helm lint` for those); entrypoint.yml is a + args: [--allow-multiple-documents, --unsafe] + # Helm chart templates are Go-template text, not parseable YAML until + # rendered (use `helm lint` for those); entrypoint.yml is a # pre-existing, never-finished Ansible scaffold stub (placeholder # "YOUR PROJECT NAME" host, references jobs/*.yml dirs that don't # exist) -- excluded rather than fixed, out of scope for this hook. @@ -36,8 +45,8 @@ repos: # convention; dns-server/safety-report.json is stray captured CLI # text (not JSON at all) from a deprecated `safety check` run. exclude: (^|/)tsconfig(\..*)?\.json$|^dns-server/safety-report\.json$ - - id: check-merge-conflict - name: Check merge conflict + - id: check-executables-have-shebangs + name: Check executables have shebangs - id: detect-private-key name: Detect private keys # Verified placeholder template (file header: "EXAMPLE / TEMPLATE @@ -45,45 +54,48 @@ repos: # scripts/gen-jwt-keys.sh"), not real key material. exclude: ^k8s/squawk-jwt-keys\.example\.yml$ - # Python linting -- args intentionally omitted so the flake8 config file - # (.flake8: E9,F63,F7,F821,F401,B) is the single source of truth instead of - # a second, drifting copy of the rule selection here. `files` is scoped to - # the same app/bins source dirs as `make lint` -- not the whole service - # tree -- so it excludes tests/, examples/, and vendored web scaffolding - # that were never vetted against these rules. - - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + # ── Secrets (mandatory, every repo) ──────────────────────────────────────── + - repo: https://github.com/gitleaks/gitleaks + rev: v8.21.2 hooks: - - id: flake8 - name: Lint Python with flake8 - additional_dependencies: [flake8-bugbear>=24.2.6] - args: [--config=.flake8] - files: ^(dns-server/app|manager/backend/app|squawk-client/bins|dhcp-server/app|ntp-server/bins)/.*\.py$ - exclude: (venv|\.venv|migrations|alembic) + - id: gitleaks - # Python code formatting - - repo: https://github.com/psf/black - rev: 23.12.1 + # ── Shell (Bash must stay 3.2-compatible — see general.md) ───────────────── + - repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.10.0 hooks: - - id: black - name: Format Python with black - args: [--line-length=120] - files: ^(dns-server/app|manager/backend/app|squawk-client/bins|dhcp-server/app|ntp-server/bins)/.*\.py$ - exclude: (venv|\.venv|migrations|alembic) - stages: [manual] # Require explicit opt-in + - id: shellcheck + args: [--severity=warning] - # Import sorting - - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + # ── Python — ruff supersedes flake8/black/isort (backend-python.md) ──────── + # `pyproject.toml` [tool.ruff.lint] carries the full canonical rule set as + # the target state. Blocking here is scoped via --select to F/E9/B -- the + # same bar flake8 already enforced (E9,F63,F7,F821,F401,B), now via ruff. + # The full canonical set (D/N/UP/ASYNC/S/I/E-full/W-full) surfaces ~1,700 + # pre-existing findings across five services that predate this change and + # are not fixable as part of a hooks-installation chore -- see the + # `ruff-full` manual hook below and the comment in pyproject.toml. Mirrors + # the mypy exception further down. + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.4 hooks: - - id: isort - name: Sort Python imports with isort - args: [--profile=black, --line-length=120] + - id: ruff + name: Lint Python with ruff (blocking subset -- F, E9, B) + args: [--select=F,E9,B, --fix] + files: ^(dns-server/app|manager/backend/app|squawk-client/bins|dhcp-server/app|ntp-server/bins)/.*\.py$ + exclude: (venv|\.venv|migrations|alembic) + - id: ruff + name: Lint Python with ruff (full canonical rule set, advisory) + files: ^(dns-server/app|manager/backend/app|squawk-client/bins|dhcp-server/app|ntp-server/bins)/.*\.py$ + exclude: (venv|\.venv|migrations|alembic) + stages: [manual] + - id: ruff-format + name: Format Python with ruff (advisory -- run via make format) files: ^(dns-server/app|manager/backend/app|squawk-client/bins|dhcp-server/app|ntp-server/bins)/.*\.py$ exclude: (venv|\.venv|migrations|alembic) - stages: [manual] # Require explicit opt-in + stages: [manual] - # Type checking -- manual stage (opt-in): dns-server and manager/backend + # ── Type checking -- manual stage (opt-in): dns-server and manager/backend # currently carry pre-existing mypy errors (Flask dynamic-attribute access, # crypto-lib stub mismatches) that predate this hook and are not yet fixed, # so this is not wired into the blocking default stage. Run explicitly with @@ -97,29 +109,63 @@ repos: pass_filenames: false stages: [manual] - # Docker linting -- includes hadolint's default DL3002 rule ("Last USER - # should not be root"), which is the non-root Dockerfile check. + # ── Go ───────────────────────────────────────────────────────────────────── + # Pinned to v1.64.8 to match .github/workflows/go-client-release.yml (v2 + # needs go1.25; this repo's .golangci.yml is still v1-schema -- see that + # file's header comment). `language: golang` builds this into an isolated, + # pinned env instead of trusting whatever golangci-lint happens to be on a + # developer's PATH (a v2 binary from e.g. a system package manager fails + # outright on the v1-schema config with "unsupported version of the + # configuration"). Scoped to squawk-client-go, matching make lint's + # existing scope -- the only Go module with an established .golangci.yml. + - repo: https://github.com/golangci/golangci-lint + rev: v1.64.8 + hooks: + - id: golangci-lint + name: Lint Go with golangci-lint + entry: bash -c 'cd squawk-client-go && golangci-lint run --config=../.golangci.yml --new-from-rev=HEAD' + pass_filenames: false + + # ── Dockerfiles ──────────────────────────────────────────────────────────── - repo: https://github.com/hadolint/hadolint - rev: v2.14.0 + rev: v2.13.1-beta hooks: - id: hadolint-docker - name: Lint Dockerfiles (includes non-root/DL3002 check) - stages: [commit, push] + name: Lint Dockerfiles - # Go linting (if golangci-lint is installed) - - repo: local + # ── GitHub Actions ───────────────────────────────────────────────────────── + - repo: https://github.com/rhysd/actionlint + rev: v1.7.4 hooks: - - id: golangci-lint - name: Lint Go with golangci-lint - entry: bash -c 'command -v golangci-lint >/dev/null && golangci-lint run squawk-client-go/... || true' - language: system - types: [go] - pass_filenames: false + - id: actionlint + # -shellcheck= disables actionlint's embedded shellcheck pass over + # `run:` blocks -- the standalone shellcheck-precommit hook above + # already covers shell content at the repo's chosen severity + # (--severity=warning); actionlint's own integration has no severity + # floor and surfaces dozens of pre-existing style-level findings + # (SC2001/SC2002/SC2086) across every workflow's run steps that + # predate this hook and aren't fixable as part of installing it. + # .ansible-lint is a stray ansible-lint config file that happens to + # live under .github/workflows/ -- not a workflow -- so it's excluded + # rather than "fixed" into looking like one. + args: ["-shellcheck="] + exclude: ^\.github/workflows/\.ansible-lint$ - # Shell script linting (if shellcheck is installed) - - id: shellcheck - name: Lint shell scripts - entry: bash -c 'command -v shellcheck >/dev/null && find . -name "*.sh" -not -path "*/.git/*" -not -path "*/venv/*" -not -path "*/.venv/*" | xargs shellcheck || true' - language: system - types: [shell] - pass_filenames: false + # ── Security (pre-push — heavier, keeps commits fast) ────────────────────── + - repo: https://github.com/PyCQA/bandit + rev: 1.8.0 + hooks: + - id: bandit + args: [-r, -ll] + files: ^(dns-server/app|manager/backend/app|squawk-client/bins|dhcp-server/app|ntp-server/bins)/.*\.py$ + exclude: (venv|\.venv|migrations|alembic) + stages: [pre-push] + + - repo: local + hooks: + - id: dockerfile-rootless + name: Dockerfile runs as non-root + entry: scripts/hooks/check-dockerfile-rootless.sh + language: script + files: (^|/)Dockerfile[^/]*$ + stages: [pre-push] diff --git a/LICENSE.md b/LICENSE.md index 4779f19d..19addd34 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -682,5 +682,3 @@ specific requirements. if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . - - diff --git a/Makefile b/Makefile index 967cf179..3868f297 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Makefile for Squawk DNS System -.PHONY: help setup test test-unit test-integration test-security test-performance clean build run stop logs shell fix-lint install-hooks smoke-test test-e2e test-functional +.PHONY: help setup test test-unit test-integration test-security test-performance clean build run stop logs shell fix-lint install-hooks verify-hooks smoke-test test-e2e test-functional # NOTE: no root-level venv/ is created by this Makefile (setup-venv only # creates dns-server/venv and squawk-client/venv) -- these resolve via PATH so @@ -8,8 +8,7 @@ PYTHON := python3 PIP := pip3 PYTEST := pytest -FLAKE8 := flake8 -BLACK := black +RUFF := ruff MYPY := mypy BANDIT := bandit SAFETY := safety @@ -23,7 +22,8 @@ help: @echo " setup-dev - Set up development environment with all tools" @echo " install - Install dependencies" @echo " install-dev - Install development dependencies" - @echo " install-hooks - Install git pre-commit hooks" + @echo " install-hooks - Install pre-commit framework + register hooks" + @echo " verify-hooks - Report whether hooks are installed and non-empty" @echo "" @echo "Testing:" @echo " smoke-test - Run fast smoke tests (pre-commit)" @@ -36,9 +36,9 @@ help: @echo " test-coverage - Run tests with coverage report" @echo "" @echo "Code Quality:" - @echo " lint - Run linting (flake8, black, isort)" - @echo " fix-lint - Fix linting errors (black, isort)" - @echo " format - Format code (black)" + @echo " lint - Run linting (ruff, golangci-lint, hadolint, ...)" + @echo " fix-lint - Fix linting errors (ruff)" + @echo " format - Format code (ruff format)" @echo " type-check - Run type checking (mypy)" @echo " security-check - Run security checks" @echo " quality-check - Run all quality checks" @@ -51,7 +51,7 @@ help: @echo "" # Setup targets -setup: setup-venv install +setup: setup-venv install install-hooks @echo "Development environment setup complete!" setup-dev: setup-venv install-dev setup-pre-commit @@ -74,14 +74,11 @@ install-dev: setup-pre-commit: install-hooks -install-hooks: - @echo "Installing pre-commit hooks..." - @if command -v pre-commit >/dev/null 2>&1; then \ - pre-commit install && pre-commit install --hook-type pre-push; \ - else \ - echo "pre-commit not found. Install with: pip install pre-commit"; \ - exit 1; \ - fi +install-hooks: ## Install pre-commit framework + register pre-commit and pre-push hooks + @./scripts/install-pre-commit.sh + +verify-hooks: ## Report whether pre-commit/pre-push hooks are installed and non-empty + @./scripts/install-pre-commit.sh --verify # Testing targets test: test-unit test-integration @@ -127,23 +124,15 @@ test-coverage: lint: @echo "=== Linting ===" @exit_code=0; \ - if command -v flake8 >/dev/null 2>&1; then \ - echo "-- flake8 --"; \ - python3 -m flake8 dns-server/app manager/backend/app squawk-client/bins dhcp-server/app ntp-server/bins --config=.flake8 || exit_code=1; \ - else \ - echo "flake8 not installed, skipping"; \ - fi; \ - if command -v black >/dev/null 2>&1; then \ - echo "-- black (check) --"; \ - black --check dns-server/app manager/backend/app squawk-client/bins dhcp-server/app ntp-server/bins --line-length=120 || true; \ - else \ - echo "black not installed, skipping"; \ - fi; \ - if command -v isort >/dev/null 2>&1; then \ - echo "-- isort (check, advisory) --"; \ - isort --check-only dns-server/app manager/backend/app squawk-client/bins dhcp-server/app ntp-server/bins --profile=black --line-length=120 2>&1 || true; \ + if command -v ruff >/dev/null 2>&1; then \ + echo "-- ruff check (blocking subset: F, E9, B) --"; \ + ruff check dns-server/app manager/backend/app squawk-client/bins dhcp-server/app ntp-server/bins --select=F,E9,B || exit_code=1; \ + echo "-- ruff check (full canonical rule set, advisory) --"; \ + ruff check dns-server/app manager/backend/app squawk-client/bins dhcp-server/app ntp-server/bins --exit-zero; \ + echo "-- ruff format (check, advisory) --"; \ + ruff format --check dns-server/app manager/backend/app squawk-client/bins dhcp-server/app ntp-server/bins || true; \ else \ - echo "isort not installed, skipping"; \ + echo "ruff not installed, skipping"; \ fi; \ if command -v golangci-lint >/dev/null 2>&1; then \ echo "-- golangci-lint (advisory) --"; \ @@ -171,15 +160,17 @@ lint: fi; \ exit $$exit_code -fix-lint: format +fix-lint: + @echo "Fixing lint errors (ruff --fix, blocking subset)..." + $(RUFF) check dns-server/app manager/backend/app squawk-client/bins dhcp-server/app ntp-server/bins --select=F,E9,B --fix format: @echo "Formatting code..." - cd dns-server && $(BLACK) app/ tests/ - cd squawk-client && $(BLACK) bins/ tests/ - cd manager/backend && $(BLACK) app/ tests/ --line-length=120 - cd dhcp-server && $(BLACK) app/ tests/ --line-length=120 - cd ntp-server && $(BLACK) bins/ tests/ --line-length=120 + $(RUFF) format dns-server/app dns-server/tests + $(RUFF) format squawk-client/bins squawk-client/tests + $(RUFF) format manager/backend/app manager/backend/tests + $(RUFF) format dhcp-server/app dhcp-server/tests + $(RUFF) format ntp-server/bins ntp-server/tests # Run per-service (`cd && mypy /`), never all at once -- # dns-server/app, manager/backend/app, and dhcp-server/app are each their own diff --git a/README.md b/README.md index 13b3be1d..a1ceddb6 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,17 @@ ``` ____ - .-~ ~-. + .-~ ~-. .--~' '~. .~' ___ '~. / (o o) \ ____ | ___ \_/ ___ | / | ( '~-----~' ) | / SQUAWK! - \ '~-._______.-~' / < + \ '~-._______.-~' / < '~. ___ .~' \ DNS-over-HTTPS with Secure Authentication for clientless applications '~-._ (__) _.-~' \____ '~~---~~' - + ``` # Squawk - DNS-over-HTTPS Proxy System @@ -452,7 +452,7 @@ Squawk provides comprehensive logging with real client IP detection and syslog s ```json { "timestamp": "2024-01-15T10:30:45.123Z", - "event_type": "dns_query", + "event_type": "dns_query", "client_ip": "203.0.113.45", "query_name": "example.com", "query_type": "A", @@ -524,7 +524,7 @@ Squawk supports Google Authenticator TOTP-based MFA for enhanced account securit ```bash # Require MFA for all users REQUIRE_MFA=true - + # Customize MFA issuer name MFA_ISSUER="Your Company DNS" ``` @@ -569,7 +569,7 @@ http://localhost:8080/dns_console/admin/sso ```json { "sso_url": "https://idp.company.com/sso/saml", - "sls_url": "https://idp.company.com/slo/saml", + "sls_url": "https://idp.company.com/slo/saml", "entity_id": "squawk-dns", "x509cert": "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----", "attribute_mapping": { @@ -594,7 +594,7 @@ http://localhost:8080/dns_console/admin/sso "admin_groups": ["cn=dns-admins,cn=groups,dc=company,dc=com"], "attribute_mapping": { "email": "mail", - "first_name": "givenName", + "first_name": "givenName", "last_name": "sn" } } @@ -607,7 +607,7 @@ http://localhost:8080/dns_console/admin/sso "client_id": "squawk-dns-client-id", "client_secret": "client-secret-here", "auth_url": "https://oauth.company.com/oauth/authorize", - "token_url": "https://oauth.company.com/oauth/token", + "token_url": "https://oauth.company.com/oauth/token", "userinfo_url": "https://oauth.company.com/oauth/userinfo", "scopes": ["openid", "profile", "email"], "redirect_uri": "https://dns.company.com/dns_console/auth/oauth/callback" @@ -935,4 +935,4 @@ See LICENSE.md in the docs folder for licensing information. - [ ] GraphQL API support - [ ] DNS-over-TLS (DoT) support - [ ] Kubernetes operator for easy deployment -- [ ] Load balancing and failover support \ No newline at end of file +- [ ] Load balancing and failover support diff --git a/dhcp-server/requirements-dev.txt b/dhcp-server/requirements-dev.txt index 7aa17f04..f7940cb2 100644 --- a/dhcp-server/requirements-dev.txt +++ b/dhcp-server/requirements-dev.txt @@ -1,6 +1,3 @@ # Development/lint tooling for DHCP server (not installed in the runtime image) -flake8>=6.1.0 -flake8-bugbear>=24.2.6 -black>=23.9.1 -isort>=5.12.0 +ruff>=0.8.4 mypy>=1.6.1 diff --git a/dns-server/TESTING.md b/dns-server/TESTING.md index 3ee3dbd9..ffaf1bfd 100644 --- a/dns-server/TESTING.md +++ b/dns-server/TESTING.md @@ -7,7 +7,7 @@ The test suite has been **streamlined** to focus on **working functionality only ## What Changed ### ❌ Removed (Moved to `tests_full_future/`) -- Tests for unimplemented IOC Manager features +- Tests for unimplemented IOC Manager features - Tests for incomplete WHOIS Manager functionality - Tests for Client Config API (database issues) - Tests for Selective DNS Routing (missing tables) @@ -16,7 +16,7 @@ The test suite has been **streamlined** to focus on **working functionality only ### ✅ Kept (Now in `tests/`) - **Core DNS functionality** - domain validation, DNS resolution -- **Authentication basics** - token generation, password validation +- **Authentication basics** - token generation, password validation - **Security features** - XSS prevention, input validation - **Health checks** - module imports, JSON handling - **Performance tests** - basic speed requirements @@ -29,7 +29,7 @@ cd /workspaces/Squawk/dns-server python -m pytest tests/ -v # ====== 18 passed in <1 second ====== -# Docker testing +# Docker testing docker build --build-arg SQUAWK_ENV=test -f dns-server/Dockerfile -t squawk-dns-server:test dns-server/ docker run --rm -e SQUAWK_ENV=test -w /app/dns-server squawk-dns-server:test # ====== 18 passed in <1 second ====== @@ -52,14 +52,14 @@ docker run --rm -e SQUAWK_ENV=test -w /app/dns-server squawk-dns-server:test - Basic error handling - Performance requirements -### Authentication (4 tests) +### Authentication (4 tests) - Token generation and validation - Password complexity rules - MFA concepts (without external deps) - Backup code generation ### Health & Integration (7 tests) -- Module import verification +- Module import verification - JSON serialization/deserialization - Environment setup validation - Async/await functionality @@ -79,4 +79,4 @@ As features are completed, tests can be gradually moved back from `tests_full_fu > **Test what works, not what you wish worked.** -This approach provides immediate feedback on regressions while avoiding the frustration of constantly failing tests for incomplete features. \ No newline at end of file +This approach provides immediate feedback on regressions while avoiding the frustration of constantly failing tests for incomplete features. diff --git a/dns-server/app/grpc_server.py b/dns-server/app/grpc_server.py index f66de2e6..e1e3b3f3 100644 --- a/dns-server/app/grpc_server.py +++ b/dns-server/app/grpc_server.py @@ -284,7 +284,7 @@ async def serve_grpc(port=50052, resolver=None, cache_manager=None, ioc_checker= ) # Create servicer - servicer = DNSQueryServicer( + DNSQueryServicer( resolver, cache_manager, ioc_checker, selective_router, manager_client, metrics_reporter ) diff --git a/dns-server/app/observability.py b/dns-server/app/observability.py index b9f8706f..b1811b99 100644 --- a/dns-server/app/observability.py +++ b/dns-server/app/observability.py @@ -7,7 +7,6 @@ import os import logging -from typing import Optional logger = logging.getLogger(__name__) diff --git a/dns-server/app/services/cert_manager.py b/dns-server/app/services/cert_manager.py index 97eba0fd..934cb61a 100644 --- a/dns-server/app/services/cert_manager.py +++ b/dns-server/app/services/cert_manager.py @@ -664,7 +664,7 @@ def verify_client_cert(self, cert_pem: str) -> VerificationResult: ) ca_cert = self._load_certificate(self.ca_cert_path) - ca_key = self._load_private_key(self.ca_key_path) + self._load_private_key(self.ca_key_path) # Verify signature try: diff --git a/dns-server/app/services/rate_limiter.py b/dns-server/app/services/rate_limiter.py index 90364018..181e4ba6 100644 --- a/dns-server/app/services/rate_limiter.py +++ b/dns-server/app/services/rate_limiter.py @@ -4,7 +4,7 @@ """ import time import logging -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Optional, Dict, Tuple from abc import ABC, abstractmethod from collections import OrderedDict diff --git a/dns-server/app/utils/jwt_verify.py b/dns-server/app/utils/jwt_verify.py index b9b1ef50..707ba9d8 100644 --- a/dns-server/app/utils/jwt_verify.py +++ b/dns-server/app/utils/jwt_verify.py @@ -21,7 +21,6 @@ import jwt as pyjwt from jwt.exceptions import ( DecodeError, - ExpiredSignatureError, InvalidAudienceError, InvalidIssuerError, InvalidSignatureError, @@ -89,8 +88,7 @@ def verify_squawk_jwt( return None # Try each key until one succeeds - last_error: Optional[Exception] = None - for kid_val, key in keys_to_try.items(): + for _kid_val, key in keys_to_try.items(): if not key: continue try: @@ -108,7 +106,7 @@ def verify_squawk_jwt( return None return payload except (InvalidSignatureError, DecodeError): - last_error = None # Signature mismatch is expected when trying multiple keys + # Signature mismatch is expected when trying multiple keys continue except (InvalidAudienceError, InvalidIssuerError, MissingRequiredClaimError) as e: logger.warning(f"JWT claim validation failed: {e}") @@ -118,7 +116,6 @@ def verify_squawk_jwt( return None except Exception as e: logger.error(f"Token validation error: {e}") - last_error = e return None # No keys succeeded diff --git a/dns-server/apt-packages.txt b/dns-server/apt-packages.txt index 96997087..1979f095 100644 --- a/dns-server/apt-packages.txt +++ b/dns-server/apt-packages.txt @@ -7,4 +7,4 @@ libldap-dev libldap2-dev libsasl2-dev python3-dev -pkg-config \ No newline at end of file +pkg-config diff --git a/dns-server/bandit-report.json b/dns-server/bandit-report.json index 4fdc32cd..197be72a 100644 --- a/dns-server/bandit-report.json +++ b/dns-server/bandit-report.json @@ -531,4 +531,4 @@ "test_name": "blacklist" } ] -} \ No newline at end of file +} diff --git a/dns-server/docker-compose.yml b/dns-server/docker-compose.yml index 75b98d7a..5b52833b 100644 --- a/dns-server/docker-compose.yml +++ b/dns-server/docker-compose.yml @@ -68,4 +68,4 @@ networks: driver: bridge volumes: - valkey-data: \ No newline at end of file + valkey-data: diff --git a/dns-server/requirements-dev.txt b/dns-server/requirements-dev.txt index d3e2dfca..82404872 100644 --- a/dns-server/requirements-dev.txt +++ b/dns-server/requirements-dev.txt @@ -4,12 +4,9 @@ pytest-asyncio>=0.23.0 pytest-cov>=4.1.0 pytest-mock>=3.12.0 pytest-xdist>=3.3.1 -black>=23.9.1 -flake8>=6.1.0 -flake8-bugbear>=24.2.6 +ruff>=0.8.4 mypy>=1.6.1 safety>=2.3.5 bandit>=1.7.5 pre-commit>=3.5.0 responses>=0.23.0 -isort>=5.12.0 diff --git a/dns-server/tests/README.md b/dns-server/tests/README.md index 46ab6df5..4cf38354 100644 --- a/dns-server/tests/README.md +++ b/dns-server/tests/README.md @@ -10,7 +10,7 @@ This is a streamlined test suite that focuses on **working functionality only**. - Basic security validation - Performance requirements -✅ **Authentication Basics** (test_authentication.py) +✅ **Authentication Basics** (test_authentication.py) - Token generation - Password complexity validation - MFA components (mocked) @@ -25,7 +25,7 @@ This is a streamlined test suite that focuses on **working functionality only**. ❌ **Unimplemented Features** - IOC Manager advanced features -- WHOIS Manager +- WHOIS Manager - Client Config API - Selective DNS Routing - Prometheus Metrics (advanced) @@ -54,4 +54,4 @@ docker run --rm -w /app/dns-server squawk-dns-server:test ## Philosophy -This test suite follows the principle of **testing what works** rather than what we wish worked. As features are fully implemented, tests can be added back from the original `tests/` directory. \ No newline at end of file +This test suite follows the principle of **testing what works** rather than what we wish worked. As features are fully implemented, tests can be added back from the original `tests/` directory. diff --git a/dns-server/tests/test_ioc_blocking.py b/dns-server/tests/test_ioc_blocking.py index 142222c2..fefd931c 100644 --- a/dns-server/tests/test_ioc_blocking.py +++ b/dns-server/tests/test_ioc_blocking.py @@ -30,21 +30,21 @@ def mock_ioc_data(): class TestIOCBlocking: """Test IOC blocking functionality""" - + def test_domain_in_blocklist(self, mock_ioc_data): """Test checking if domain is in blocklist""" blocklist = set(mock_ioc_data['malicious_domains']) - + assert 'malware.example.com' in blocklist assert 'safe.example.com' not in blocklist - + def test_ip_in_blocklist(self, mock_ioc_data): """Test checking if IP is in blocklist""" blocklist = set(mock_ioc_data['malicious_ips']) - + assert '192.0.2.1' in blocklist assert '93.184.216.34' not in blocklist - + @patch('requests.get') def test_fetch_ioc_feed(self, mock_get, mock_ioc_data): """Test fetching IOC feed from URL""" @@ -52,33 +52,33 @@ def test_fetch_ioc_feed(self, mock_get, mock_ioc_data): mock_response.status_code = 200 mock_response.text = '\n'.join(mock_ioc_data['malicious_domains']) mock_get.return_value = mock_response - + # Simulate fetching feed response = mock_get('https://example.com/ioc-feed.txt') assert response.status_code == 200 - + domains = response.text.split('\n') assert len(domains) == 3 assert 'malware.example.com' in domains - + def test_parse_ioc_feed(self, mock_ioc_data): """Test parsing IOC feed data""" feed_text = '\n'.join(mock_ioc_data['malicious_domains']) - + # Parse feed domains = [line.strip() for line in feed_text.split('\n') if line.strip()] - + assert len(domains) == 3 assert all(domain in mock_ioc_data['malicious_domains'] for domain in domains) class TestIOCFeedManagement: """Test IOC feed management""" - + def test_add_ioc_feed(self): """Test adding IOC feed""" feeds = [] - + new_feed = { 'id': 1, 'name': 'Test Feed', @@ -86,11 +86,11 @@ def test_add_ioc_feed(self): 'type': 'domain', 'active': True } - + feeds.append(new_feed) assert len(feeds) == 1 assert feeds[0]['name'] == 'Test Feed' - + def test_update_ioc_feed(self): """Test updating IOC feed""" feed = { @@ -99,93 +99,93 @@ def test_update_ioc_feed(self): 'url': 'https://example.com/feed.txt', 'active': True } - + # Update feed feed['active'] = False feed['url'] = 'https://example.com/new-feed.txt' - + assert feed['active'] == False assert feed['url'] == 'https://example.com/new-feed.txt' - + def test_remove_ioc_feed(self): """Test removing IOC feed""" feeds = [ {'id': 1, 'name': 'Feed 1'}, {'id': 2, 'name': 'Feed 2'} ] - + # Remove feed feeds = [f for f in feeds if f['id'] != 1] - + assert len(feeds) == 1 assert feeds[0]['id'] == 2 class TestIOCQueryBlocking: """Test DNS query blocking based on IOC""" - + def test_block_malicious_domain(self, mock_ioc_data): """Test blocking query for malicious domain""" blocklist = set(mock_ioc_data['malicious_domains']) - + query_domain = 'malware.example.com' - + # Check if should be blocked should_block = query_domain in blocklist assert should_block == True - + def test_allow_safe_domain(self, mock_ioc_data): """Test allowing query for safe domain""" blocklist = set(mock_ioc_data['malicious_domains']) - + query_domain = 'google.com' - + # Check if should be blocked should_block = query_domain in blocklist assert should_block == False - + def test_subdomain_blocking(self, mock_ioc_data): """Test blocking subdomains of malicious domains""" blocklist = set(mock_ioc_data['malicious_domains']) - + # Add wildcard support query_domain = 'sub.malware.example.com' - + # Check if parent domain is in blocklist parts = query_domain.split('.') should_block = False - + for i in range(len(parts)): potential_domain = '.'.join(parts[i:]) if potential_domain in blocklist: should_block = True break - + assert should_block == True class TestIOCMetrics: """Test IOC blocking metrics""" - + def test_count_blocked_queries(self): """Test counting blocked queries""" blocked_queries = [] - + # Simulate blocking queries blocked_queries.append({ 'domain': 'malware.example.com', 'timestamp': datetime.utcnow(), 'reason': 'IOC blocklist' }) - + blocked_queries.append({ 'domain': 'phishing.example.com', 'timestamp': datetime.utcnow(), 'reason': 'IOC blocklist' }) - + assert len(blocked_queries) == 2 - + def test_ioc_feed_statistics(self): """Test IOC feed statistics""" feed_stats = { @@ -194,6 +194,6 @@ def test_ioc_feed_statistics(self): 'ips': 200, 'last_updated': datetime.utcnow() } - + assert feed_stats['total_entries'] == 1000 assert feed_stats['domains'] + feed_stats['ips'] == feed_stats['total_entries'] diff --git a/dns-server/tests_full_future/__init__.py b/dns-server/tests_full_future/__init__.py index 38075b74..4c460df8 100644 --- a/dns-server/tests_full_future/__init__.py +++ b/dns-server/tests_full_future/__init__.py @@ -1 +1 @@ -# DNS Server Tests Package \ No newline at end of file +# DNS Server Tests Package diff --git a/dns-server/tests_full_future/conftest.py b/dns-server/tests_full_future/conftest.py index afede1dc..3fe2a10a 100644 --- a/dns-server/tests_full_future/conftest.py +++ b/dns-server/tests_full_future/conftest.py @@ -394,14 +394,14 @@ def mock_dns_handler(): handler.headers = {'Authorization': 'Bearer test-token-123456789'} handler.path = '/dns-query?name=example.com&type=A' handler.client_address = ('127.0.0.1', 12345) - + # Mock methods handler.send_response = Mock() handler.send_header = Mock() handler.end_headers = Mock() handler.wfile = Mock() handler.wfile.write = Mock() - + return handler @pytest.fixture @@ -410,11 +410,11 @@ def mock_dns_resolver(): with patch('dns.resolver.Resolver') as mock_resolver: mock_answer = Mock() mock_answer.to_text.return_value = '93.184.216.34' - + mock_resolver_instance = Mock() mock_resolver_instance.resolve.return_value = [mock_answer] mock_resolver.return_value = mock_resolver_instance - + yield mock_resolver_instance @pytest.fixture @@ -527,4 +527,4 @@ def _bypass_feed_url_ssrf_check(): yield return with patch.object(ioc_ingestion_service, "_assert_feed_url_safe", new=AsyncMock()): - yield \ No newline at end of file + yield diff --git a/dns-server/tests_full_future/test_client_config_api.py b/dns-server/tests_full_future/test_client_config_api.py index 004c0fe1..282a8fa8 100644 --- a/dns-server/tests_full_future/test_client_config_api.py +++ b/dns-server/tests_full_future/test_client_config_api.py @@ -12,7 +12,7 @@ from client_config_api import ClientConfigManager class TestClientConfigManager: - + @pytest.fixture def config_manager(self, temp_db): """Create client config manager instance with test database. @@ -21,20 +21,20 @@ def config_manager(self, temp_db): """ db_url = f"sqlite://{temp_db._uri[9:]}" # Extract path from DAL URI return ClientConfigManager(db_url) - + def test_create_deployment_domain(self, config_manager): """Test creating a new deployment domain""" result = config_manager.create_deployment_domain( - "test-domain", - "Test deployment domain", + "test-domain", + "Test deployment domain", "test_admin" ) - + assert result['success'] is True assert result['name'] == "test-domain" assert 'jwt_token' in result assert 'id' in result - + # Verify JWT token is valid token = result['jwt_token'] decoded = jwt.decode( @@ -43,32 +43,32 @@ def test_create_deployment_domain(self, config_manager): ) assert decoded['domain'] == "test-domain" assert decoded['type'] == 'deployment_domain' - + def test_create_duplicate_domain(self, config_manager): """Test creating duplicate deployment domain fails""" # Create first domain result1 = config_manager.create_deployment_domain("duplicate", "First") assert result1['success'] is True - + # Attempt duplicate result2 = config_manager.create_deployment_domain("duplicate", "Second") assert result2['success'] is False assert 'error' in result2 - + def test_rollover_domain_jwt(self, config_manager): """Test JWT token rollover for domain""" # Create domain domain_result = config_manager.create_deployment_domain("rollover-test", "Test domain") domain_id = domain_result['id'] old_jwt = domain_result['jwt_token'] - + # Rollover JWT rollover_result = config_manager.rollover_domain_jwt(domain_id, "admin_user") - + assert rollover_result['success'] is True assert 'new_jwt' in rollover_result assert rollover_result['new_jwt'] != old_jwt - + # Verify new JWT is valid new_token = rollover_result['new_jwt'] decoded = jwt.decode( @@ -76,13 +76,13 @@ def test_rollover_domain_jwt(self, config_manager): audience=config_manager.audience, issuer=config_manager.issuer, ) assert decoded['domain'] == "rollover-test" - + def test_create_client_config(self, config_manager, mock_client_config): """Test creating client configuration""" # Create domain first domain_result = config_manager.create_deployment_domain("config-test", "Config test") domain_id = domain_result['id'] - + # Create config config_result = config_manager.create_client_config( "test-config", @@ -91,19 +91,19 @@ def test_create_client_config(self, config_manager, mock_client_config): "Test configuration", "test_creator" ) - + assert config_result['success'] is True assert 'config_id' in config_result assert config_result['version'] == 1 - + def test_create_invalid_client_config(self, config_manager): """Test creating client config with invalid data""" domain_result = config_manager.create_deployment_domain("invalid-config", "Test") domain_id = domain_result['id'] - + # Invalid config (missing required fields) invalid_config = {'incomplete': 'config'} - + config_result = config_manager.create_client_config( "invalid-config", domain_id, @@ -111,39 +111,39 @@ def test_create_invalid_client_config(self, config_manager): "Invalid config test", "test_creator" ) - + assert config_result['success'] is False assert 'error' in config_result - + def test_update_client_config(self, config_manager, mock_client_config): """Test updating existing client configuration""" # Create domain and initial config domain_result = config_manager.create_deployment_domain("update-test", "Update test") domain_id = domain_result['id'] - + config_result = config_manager.create_client_config( "update-config", domain_id, mock_client_config, "Initial", "creator" ) config_id = config_result['config_id'] - + # Update config updated_config = mock_client_config.copy() updated_config['dns_port'] = 5353 # Change port updated_config['cache_ttl'] = 600 # Change TTL - + update_result = config_manager.update_client_config( config_id, updated_config, "Updated configuration", "updater" ) - + assert update_result['success'] is True assert update_result['version'] == 2 - + def test_register_client(self, config_manager): """Test client registration""" # Create domain domain_result = config_manager.create_deployment_domain("client-test", "Client test") domain_jwt = domain_result['jwt_token'] - + # Register client register_result = config_manager.register_client( "client-123", @@ -153,29 +153,29 @@ def test_register_client(self, config_manager): "v2.0.0", "Linux Ubuntu 22.04" ) - + assert register_result['success'] is True assert 'client_record_id' in register_result assert register_result['domain_name'] == "client-test" - + def test_register_client_invalid_jwt(self, config_manager): """Test client registration with invalid JWT""" register_result = config_manager.register_client( "client-invalid", "invalid.jwt.token", - "test-hostname", + "test-hostname", "192.168.1.101" ) - + assert register_result['success'] is False assert 'invalid' in register_result['error'].lower() - + def test_register_client_with_user_token(self, config_manager, sample_token_data): """Test client registration with user authentication""" # Create domain domain_result = config_manager.create_deployment_domain("auth-test", "Auth test") domain_jwt = domain_result['jwt_token'] - + # Register client with user token register_result = config_manager.register_client( "auth-client-123", @@ -186,153 +186,153 @@ def test_register_client_with_user_token(self, config_manager, sample_token_data "Linux", user_token=sample_token_data['token'] ) - + assert register_result['success'] is True assert register_result['domain_name'] == "auth-test" - + def test_pull_client_config(self, config_manager, mock_client_config): """Test pulling client configuration""" # Create domain and config domain_result = config_manager.create_deployment_domain("pull-test", "Pull test") domain_id = domain_result['id'] domain_jwt = domain_result['jwt_token'] - + config_result = config_manager.create_client_config( "default", domain_id, mock_client_config, "Default config", "creator" ) - + # Register client register_result = config_manager.register_client( "pull-client-123", domain_jwt, "pull-host", "192.168.1.103" ) - + # Pull configuration pull_result = config_manager.pull_client_config( "pull-client-123", domain_jwt ) - + assert pull_result['success'] is True assert 'config' in pull_result assert pull_result['config']['server_url'] == mock_client_config['server_url'] assert pull_result['config']['dns_port'] == mock_client_config['dns_port'] assert pull_result['version'] == 1 assert pull_result['config_name'] == "default" - + def test_pull_config_with_user_auth(self, config_manager, mock_client_config, sample_token_data): """Test pulling config with user authentication""" # Create domain and config domain_result = config_manager.create_deployment_domain("auth-pull", "Auth pull test") domain_id = domain_result['id'] domain_jwt = domain_result['jwt_token'] - + config_result = config_manager.create_client_config( "default", domain_id, mock_client_config, "Auth config", "creator" ) - + # Register client with user token register_result = config_manager.register_client( "auth-pull-client", domain_jwt, "auth-pull-host", "192.168.1.104", user_token=sample_token_data['token'] ) - + # Pull config with user token pull_result = config_manager.pull_client_config( "auth-pull-client", domain_jwt, sample_token_data['token'] ) - + assert pull_result['success'] is True assert 'config' in pull_result - + def test_pull_config_unregistered_client(self, config_manager): """Test pulling config for unregistered client""" domain_result = config_manager.create_deployment_domain("unreg-test", "Unregistered test") domain_jwt = domain_result['jwt_token'] - + pull_result = config_manager.pull_client_config( "unregistered-client", domain_jwt ) - + assert pull_result['success'] is False assert 'not registered' in pull_result['error'].lower() - + def test_assign_config_to_client(self, config_manager, mock_client_config): """Test assigning specific configuration to client""" # Create domain domain_result = config_manager.create_deployment_domain("assign-test", "Assign test") domain_id = domain_result['id'] domain_jwt = domain_result['jwt_token'] - + # Create two configs config1 = config_manager.create_client_config( "config-1", domain_id, mock_client_config, "Config 1", "creator" ) - + config2_data = mock_client_config.copy() config2_data['dns_port'] = 5353 config2 = config_manager.create_client_config( "config-2", domain_id, config2_data, "Config 2", "creator" ) - + # Register client register_result = config_manager.register_client( "assign-client", domain_jwt, "assign-host", "192.168.1.105" ) - + # Assign specific config to client assign_result = config_manager.assign_config_to_client( "assign-client", config2['config_id'], "admin" ) - + assert assign_result['success'] is True - + # Pull config and verify it's config-2 pull_result = config_manager.pull_client_config("assign-client", domain_jwt) assert pull_result['success'] is True assert pull_result['config']['dns_port'] == 5353 assert pull_result['config_name'] == "config-2" - + def test_get_domain_clients(self, config_manager): """Test getting all clients in a domain""" # Create domain domain_result = config_manager.create_deployment_domain("clients-test", "Clients test") domain_id = domain_result['id'] domain_jwt = domain_result['jwt_token'] - + # Register multiple clients clients = ["client-1", "client-2", "client-3"] for client_id in clients: config_manager.register_client( client_id, domain_jwt, f"host-{client_id}", f"192.168.1.{clients.index(client_id) + 110}" ) - + # Get clients domain_clients = config_manager.get_domain_clients(domain_id) - + assert len(domain_clients) == 3 client_ids = [c['client_id'] for c in domain_clients] for client_id in clients: assert client_id in client_ids - + def test_get_client_stats(self, config_manager, mock_client_config): """Test getting client configuration statistics""" # Create some test data domain_result = config_manager.create_deployment_domain("stats-test", "Stats test") domain_id = domain_result['id'] domain_jwt = domain_result['jwt_token'] - + # Create config config_manager.create_client_config( "stats-config", domain_id, mock_client_config, "Stats config", "creator" ) - + # Register client config_manager.register_client( "stats-client", domain_jwt, "stats-host", "192.168.1.120" ) - + # Get stats stats = config_manager.get_client_stats() - + assert 'domains' in stats assert 'clients' in stats assert 'configurations' in stats @@ -342,26 +342,26 @@ def test_get_client_stats(self, config_manager, mock_client_config): assert stats['clients']['active'] >= 1 assert stats['configurations']['total'] >= 1 assert stats['configurations']['active'] >= 1 - + def test_cleanup_inactive_clients(self, config_manager): """Test cleanup of inactive clients""" # Create domain and register client domain_result = config_manager.create_deployment_domain("cleanup-test", "Cleanup test") domain_jwt = domain_result['jwt_token'] - + register_result = config_manager.register_client( "inactive-client", domain_jwt, "inactive-host", "192.168.1.130" ) - + # Cleanup with 0 days (removes all) deleted = config_manager.cleanup_inactive_clients(inactive_days=0) - + assert deleted >= 1 - + # Verify client was removed clients = config_manager.get_domain_clients(domain_result['id']) assert len(clients) == 0 - + def test_expired_jwt_rejection(self, config_manager): """Test that expired JWT tokens are rejected""" # Create a genuinely expired ES256 domain token (signed with the @@ -377,15 +377,15 @@ def test_expired_jwt_rejection(self, config_manager): expired_jwt = jwt.encode( expired_payload, config_manager.private_key, algorithm='ES256' ) - + # Try to register client with expired JWT register_result = config_manager.register_client( "expired-client", expired_jwt, "expired-host", "192.168.1.140" ) - + assert register_result['success'] is False assert 'invalid' in register_result['error'].lower() or 'expired' in register_result['error'].lower() - + def test_config_validation(self, config_manager): """Test configuration data validation""" test_cases = [ @@ -395,20 +395,20 @@ def test_config_validation(self, config_manager): 'dns_port': 53, 'cache_enabled': True }, True), - + # Missing required field ({ 'dns_port': 53, 'cache_enabled': True }, False), - + # Invalid server URL ({ 'server_url': 'not-a-url', 'dns_port': 53, 'cache_enabled': True }, False), - + # Invalid port ({ 'server_url': 'https://dns.example.com', @@ -416,11 +416,11 @@ def test_config_validation(self, config_manager): 'cache_enabled': True }, False) ] - + for config_data, should_be_valid in test_cases: is_valid = config_manager._validate_config_data(config_data) assert is_valid == should_be_valid, f"Validation failed for {config_data}" - + def test_certificate_subject_extraction(self, config_manager): """Test extracting CN from certificate subject DN""" test_cases = [ @@ -430,11 +430,11 @@ def test_certificate_subject_extraction(self, config_manager): ("invalid-dn-format", None), ("", None) ] - + for subject_dn, expected_cn in test_cases: extracted_cn = config_manager._extract_cn_from_subject(subject_dn) assert extracted_cn == expected_cn, f"CN extraction failed for {subject_dn}" - + def test_user_token_verification_with_mtls(self, config_manager, sample_token_data): """Test user token verification with mTLS certificate""" from penguin_dal import DB @@ -446,10 +446,10 @@ def test_user_token_verification_with_mtls(self, config_manager, sample_token_da result = config_manager._verify_user_token( db, sample_token_data['token'], f"CN={sample_token_data['token']},O=Test" ) - + assert result['valid'] is True assert result['token_id'] == sample_token_data['token_id'] - + # Invalid certificate subject not matching token result = config_manager._verify_user_token( db, sample_token_data['token'], "CN=different-name,O=Test" @@ -458,62 +458,62 @@ def test_user_token_verification_with_mtls(self, config_manager, sample_token_da # This test depends on implementation - may pass or fail based on exact logic # The key is that the function handles certificate validation assert 'valid' in result - + def test_config_history_tracking(self, config_manager, mock_client_config): """Test that configuration changes are tracked in history""" # Create domain and config domain_result = config_manager.create_deployment_domain("history-test", "History test") domain_id = domain_result['id'] - + config_result = config_manager.create_client_config( "history-config", domain_id, mock_client_config, "Initial config", "creator" ) config_id = config_result['config_id'] - + # Update config multiple times for i in range(3): updated_config = mock_client_config.copy() updated_config['cache_ttl'] = 300 + (i * 100) - + config_manager.update_client_config( config_id, updated_config, f"Update {i+1}", "updater" ) - + # Verify final version # (This would require a method to get config history, which might not be implemented) # For now, just verify the update succeeded assert True # Placeholder - would check history if API existed - + @pytest.mark.asyncio async def test_concurrent_registrations(self, config_manager): """Test concurrent client registrations""" # Create domain domain_result = config_manager.create_deployment_domain("concurrent-test", "Concurrent test") domain_jwt = domain_result['jwt_token'] - + # Define registration tasks async def register_client(client_id, ip): return config_manager.register_client( client_id, domain_jwt, f"host-{client_id}", ip ) - + # Run concurrent registrations tasks = [ register_client(f"concurrent-{i}", f"192.168.1.{150+i}") for i in range(5) ] - + results = await asyncio.gather(*tasks, return_exceptions=True) - + # All should succeed assert len(results) == 5 successful = sum(1 for r in results if isinstance(r, dict) and r.get('success')) assert successful == 5 - + # Verify all clients were registered clients = config_manager.get_domain_clients(domain_result['id']) assert len(clients) == 5 - + def test_default_roles_creation(self, config_manager): """Test that default roles are created during initialization""" from penguin_dal import DB @@ -532,4 +532,4 @@ def test_default_roles_creation(self, config_manager): domain_admin = db(db.config_role.name == 'Domain-Admin').select().first() assert domain_admin is not None - assert 'rollover_jwt' in domain_admin.permissions \ No newline at end of file + assert 'rollover_jwt' in domain_admin.permissions diff --git a/dns-server/tests_full_future/test_client_config_tenant.py b/dns-server/tests_full_future/test_client_config_tenant.py index 12b5ba13..b4a86af2 100644 --- a/dns-server/tests_full_future/test_client_config_tenant.py +++ b/dns-server/tests_full_future/test_client_config_tenant.py @@ -24,7 +24,7 @@ def config_manager(self, temp_db, test_jwt_secret): def test_cannot_overwrite_client_from_different_domain(self, config_manager, mock_client_config): """Regression: client_id registered in domain A cannot be hijacked using domain B's JWT. - + This tests the IDOR fix where register_client must scope the client_instance lookup by BOTH client_id AND domain_id to prevent cross-domain overwrites. """ @@ -83,7 +83,7 @@ def test_cannot_overwrite_client_from_different_domain(self, config_manager, moc def test_cannot_assign_config_across_domains(self, config_manager, mock_client_config): """Regression: config from domain A cannot be assigned to a client in domain B. - + This tests the IDOR fix where assign_config_to_client must verify that the config's domain_id matches the client's domain_id. """ diff --git a/dns-server/tests_full_future/test_ioc_manager.py b/dns-server/tests_full_future/test_ioc_manager.py index 0a6398a7..2ad23fa6 100644 --- a/dns-server/tests_full_future/test_ioc_manager.py +++ b/dns-server/tests_full_future/test_ioc_manager.py @@ -11,129 +11,129 @@ from ioc_manager import IOCManager class TestIOCManager: - + @pytest.fixture def ioc_manager(self, temp_db): """Create IOC manager instance with test database""" db_url = f"sqlite://{temp_db._uri[9:]}" # Extract path from DAL URI return IOCManager(db_url) - + @pytest.mark.asyncio async def test_check_domain_clean(self, ioc_manager): """Test checking clean domain not in IOC feeds""" should_block, reason = await ioc_manager.check_domain('clean-domain.com') - + assert should_block is False assert reason == "Not blocked" - + @pytest.mark.asyncio async def test_check_domain_blocked(self, ioc_manager, mock_ioc_feeds): """Test checking domain that should be blocked""" # Add test IOC data await ioc_manager.update_feed_from_content( - "Test Feed", - mock_ioc_feeds[0]['content'], - mock_ioc_feeds[0]['feed_type'], + "Test Feed", + mock_ioc_feeds[0]['content'], + mock_ioc_feeds[0]['feed_type'], mock_ioc_feeds[0]['format'] ) - + should_block, reason = await ioc_manager.check_domain('malware.example.com') - + assert should_block is True assert 'threat intelligence' in reason.lower() - + @pytest.mark.asyncio async def test_check_ip_blocked(self, ioc_manager): """Test checking IP address that should be blocked""" # Add malicious IP to IOC database await ioc_manager.update_feed_from_content( - "Malicious IPs", - "192.0.2.100\n203.0.113.50\n198.51.100.25\n", - "ip", + "Malicious IPs", + "192.0.2.100\n203.0.113.50\n198.51.100.25\n", + "ip", "txt" ) - + should_block, reason = await ioc_manager.check_ip('192.0.2.100') - + assert should_block is True assert 'threat intelligence' in reason.lower() - + @pytest.mark.asyncio async def test_check_ip_clean(self, ioc_manager): """Test checking clean IP not in IOC feeds""" should_block, reason = await ioc_manager.check_ip('8.8.8.8') - + assert should_block is False assert reason == "Not blocked" - + @pytest.mark.asyncio async def test_add_override_allow(self, ioc_manager): """Test adding override to allow blocked domain""" # First add domain to IOC feeds await ioc_manager.update_feed_from_content( - "Test Feed", - "blocked-domain.com\n", - "domain", + "Test Feed", + "blocked-domain.com\n", + "domain", "txt" ) - + # Verify it's blocked should_block, _ = await ioc_manager.check_domain('blocked-domain.com', token_id=1) assert should_block is True - + # Add override success = await ioc_manager.add_override( - 1, 'blocked-domain.com', 'domain', 'allow', + 1, 'blocked-domain.com', 'domain', 'allow', 'Testing override', 'test_user' ) assert success is True - + # Now it should be allowed should_block, reason = await ioc_manager.check_domain('blocked-domain.com', token_id=1) assert should_block is False assert 'override' in reason.lower() - + @pytest.mark.asyncio async def test_add_override_block(self, ioc_manager): """Test adding override to block clean domain""" # Verify domain is clean should_block, _ = await ioc_manager.check_domain('clean-domain.com', token_id=1) assert should_block is False - + # Add block override success = await ioc_manager.add_override( - 1, 'clean-domain.com', 'domain', 'block', + 1, 'clean-domain.com', 'domain', 'block', 'Custom block for testing', 'test_user' ) assert success is True - + # Now it should be blocked should_block, reason = await ioc_manager.check_domain('clean-domain.com', token_id=1) assert should_block is True assert 'override' in reason.lower() - + @pytest.mark.asyncio async def test_remove_override(self, ioc_manager): """Test removing an override""" # Add override first await ioc_manager.add_override( - 1, 'test-override.com', 'domain', 'block', + 1, 'test-override.com', 'domain', 'block', 'Test override', 'test_user' ) - + # Verify override exists should_block, _ = await ioc_manager.check_domain('test-override.com', token_id=1) assert should_block is True - + # Remove override success = await ioc_manager.remove_override(1, 'test-override.com', 'domain') assert success is True - + # Should be clean again should_block, reason = await ioc_manager.check_domain('test-override.com', token_id=1) assert should_block is False assert 'override' not in reason.lower() - + @pytest.mark.asyncio async def test_get_overrides(self, ioc_manager): """Test getting user's overrides""" @@ -141,51 +141,51 @@ async def test_get_overrides(self, ioc_manager): await ioc_manager.add_override(1, 'override1.com', 'domain', 'allow', 'Test 1', 'test_user') await ioc_manager.add_override(1, 'override2.com', 'domain', 'block', 'Test 2', 'test_user') await ioc_manager.add_override(1, '192.0.2.100', 'ip', 'allow', 'Test IP', 'test_user') - + overrides = await ioc_manager.get_overrides(1) - + assert len(overrides) == 3 - + # Check that all overrides are returned indicators = [o['indicator'] for o in overrides] assert 'override1.com' in indicators assert 'override2.com' in indicators assert '192.0.2.100' in indicators - + @pytest.mark.asyncio async def test_expired_override(self, ioc_manager): """Test that expired overrides are ignored""" # Add expired override expired_time = datetime.now() - timedelta(hours=1) await ioc_manager.add_override( - 1, 'expired-override.com', 'domain', 'block', + 1, 'expired-override.com', 'domain', 'block', 'Expired override', 'test_user', expired_time ) - + # Should not be blocked due to expired override should_block, reason = await ioc_manager.check_domain('expired-override.com', token_id=1) assert should_block is False assert 'override' not in reason.lower() - + @pytest.mark.asyncio async def test_update_feed_txt_format(self, ioc_manager): """Test updating IOC feed with TXT format""" content = "malware1.com\nmalware2.com\nphishing.example.org\n" - + result = await ioc_manager.update_feed_from_content( "TXT Feed", content, "domain", "txt" ) - + assert result['success'] is True assert result['indicators_added'] == 3 - + # Verify domains are blocked should_block, _ = await ioc_manager.check_domain('malware1.com') assert should_block is True - + should_block, _ = await ioc_manager.check_domain('phishing.example.org') assert should_block is True - + @pytest.mark.asyncio async def test_update_feed_csv_format(self, ioc_manager): """Test updating IOC feed with CSV format""" @@ -193,31 +193,31 @@ async def test_update_feed_csv_format(self, ioc_manager): content += "badware.com,domain,malware,95\n" content += "192.0.2.200,ip,botnet,80\n" content += "evil.example.net,domain,phishing,90\n" - + result = await ioc_manager.update_feed_from_content( "CSV Feed", content, "mixed", "csv" ) - + assert result['success'] is True assert result['indicators_added'] == 3 - + # Verify indicators are blocked should_block, _ = await ioc_manager.check_domain('badware.com') assert should_block is True - + should_block, _ = await ioc_manager.check_ip('192.0.2.200') assert should_block is True - + @pytest.mark.asyncio async def test_update_feed_json_format(self, ioc_manager): """Test updating IOC feed with JSON format""" import json - + feed_data = { "indicators": [ { "indicator": "json-malware.com", - "type": "domain", + "type": "domain", "threat_type": "malware", "confidence": 95 }, @@ -229,23 +229,23 @@ async def test_update_feed_json_format(self, ioc_manager): } ] } - + content = json.dumps(feed_data) - + result = await ioc_manager.update_feed_from_content( "JSON Feed", content, "mixed", "json" ) - + assert result['success'] is True assert result['indicators_added'] == 2 - + # Verify indicators are blocked should_block, _ = await ioc_manager.check_domain('json-malware.com') assert should_block is True - + should_block, _ = await ioc_manager.check_ip('198.51.100.100') assert should_block is True - + @pytest.mark.asyncio async def test_update_all_feeds(self, ioc_manager): """Test updating all registered feeds""" @@ -256,26 +256,26 @@ async def test_update_all_feeds(self, ioc_manager): mock_response.text = AsyncMock(return_value="threat1.com\nthreat2.com\n") mock_response.status = 200 mock_get.return_value.__aenter__.return_value = mock_response - + # First register a feed await ioc_manager.register_feed( "Test Online Feed", - "https://example.com/threats.txt", - "domain", + "https://example.com/threats.txt", + "domain", "txt", update_frequency_hours=6 ) - + # Update all feeds result = await ioc_manager.update_all_feeds() - + assert result['success'] is True assert result['feeds_updated'] == 1 - + # Verify indicators were added should_block, _ = await ioc_manager.check_domain('threat1.com') assert should_block is True - + @pytest.mark.asyncio async def test_feed_update_frequency(self, ioc_manager): """Test that feeds respect update frequency""" @@ -284,25 +284,25 @@ async def test_feed_update_frequency(self, ioc_manager): "Frequency Test Feed", "https://example.com/threats.txt", "domain", - "txt", + "txt", update_frequency_hours=1 ) - + with patch('aiohttp.ClientSession.get') as mock_get: mock_response = Mock() mock_response.text = AsyncMock(return_value="freq-test.com\n") mock_response.status = 200 mock_get.return_value.__aenter__.return_value = mock_response - + # First update result1 = await ioc_manager.update_all_feeds() assert result1['feeds_updated'] == 1 - + # Immediate second update should skip (frequency not met) result2 = await ioc_manager.update_all_feeds() assert result2['feeds_updated'] == 0 assert 'skipped' in result2 - + @pytest.mark.asyncio async def test_feed_registration(self, ioc_manager): """Test registering new IOC feed""" @@ -314,15 +314,15 @@ async def test_feed_registration(self, ioc_manager): update_frequency_hours=12, enabled=True ) - + assert result['success'] is True assert 'feed_id' in result - + # Verify feed was registered stats = await ioc_manager.get_stats() assert stats['feeds']['total'] == 1 assert stats['feeds']['enabled'] == 1 - + @pytest.mark.asyncio async def test_feed_disable_enable(self, ioc_manager): """Test disabling and enabling feeds""" @@ -330,50 +330,50 @@ async def test_feed_disable_enable(self, ioc_manager): reg_result = await ioc_manager.register_feed( "Disable Test", "https://test.com/feed.txt", - "domain", + "domain", "txt" ) feed_id = reg_result['feed_id'] - + # Disable feed disable_result = await ioc_manager.set_feed_enabled(feed_id, False) assert disable_result['success'] is True - + # Enable feed enable_result = await ioc_manager.set_feed_enabled(feed_id, True) assert enable_result['success'] is True - + # Verify final state stats = await ioc_manager.get_stats() assert stats['feeds']['enabled'] == 1 - + @pytest.mark.asyncio async def test_get_stats(self, ioc_manager): """Test IOC statistics collection""" # Add some test data await ioc_manager.update_feed_from_content( - "Stats Test", - "stats1.com\nstats2.com\nstats3.com\n", - "domain", + "Stats Test", + "stats1.com\nstats2.com\nstats3.com\n", + "domain", "txt" ) - + await ioc_manager.add_override( - 1, 'override-stats.com', 'domain', 'block', + 1, 'override-stats.com', 'domain', 'block', 'Stats test', 'test_user' ) - + stats = await ioc_manager.get_stats() - + assert 'feeds' in stats assert 'indicators' in stats assert 'overrides' in stats assert 'recent_activity' in stats - + assert stats['indicators']['total'] >= 3 assert stats['overrides']['total'] >= 1 assert stats['feeds']['total'] >= 1 - + @pytest.mark.asyncio async def test_wildcard_domain_matching(self, ioc_manager): """Test wildcard domain matching in IOC feeds""" @@ -384,18 +384,18 @@ async def test_wildcard_domain_matching(self, ioc_manager): "domain", "txt" ) - + # Test wildcard matches should_block, _ = await ioc_manager.check_domain('subdomain.malware-family.com') assert should_block is True - + should_block, _ = await ioc_manager.check_domain('test.phishing-kit.org') assert should_block is True - + # Test non-matches should_block, _ = await ioc_manager.check_domain('malware-family.com.evil.org') assert should_block is False - + @pytest.mark.asyncio async def test_cidr_ip_matching(self, ioc_manager): """Test CIDR block matching for IP addresses""" @@ -406,21 +406,21 @@ async def test_cidr_ip_matching(self, ioc_manager): "ip", "txt" ) - + # Test IPs in CIDR blocks should_block, _ = await ioc_manager.check_ip('192.0.2.50') assert should_block is True - + should_block, _ = await ioc_manager.check_ip('203.0.113.10') assert should_block is True - + # Test IPs outside CIDR blocks should_block, _ = await ioc_manager.check_ip('192.0.3.50') assert should_block is False - + should_block, _ = await ioc_manager.check_ip('203.0.114.10') assert should_block is False - + @pytest.mark.asyncio async def test_feed_update_error_handling(self, ioc_manager): """Test error handling during feed updates""" @@ -431,45 +431,45 @@ async def test_feed_update_error_handling(self, ioc_manager): "domain", "txt" ) - + # Mock network error with patch('aiohttp.ClientSession.get') as mock_get: mock_get.side_effect = Exception("Network error") - + result = await ioc_manager.update_all_feeds() - + # Should handle error gracefully assert 'error' in result or result['success'] is False - + @pytest.mark.asyncio async def test_malformed_feed_content(self, ioc_manager): """Test handling of malformed feed content""" # Test malformed CSV malformed_csv = "indicator,type\nno_type_column.com\n" - + result = await ioc_manager.update_feed_from_content( "Malformed CSV", malformed_csv, "domain", "csv" ) - + # Should handle gracefully and parse what it can assert result['success'] is True assert result.get('warnings') or result.get('indicators_added', 0) >= 0 - + # Test malformed JSON malformed_json = '{"indicators": [{"indicator": "test.com", missing_type}]}' - + result = await ioc_manager.update_feed_from_content( "Malformed JSON", malformed_json, "domain", "json" ) - + # Should handle JSON parse error assert 'error' in result or result['success'] is False - + @pytest.mark.asyncio async def test_indicator_confidence_scoring(self, ioc_manager): """Test confidence scoring for indicators""" import json - + # Add indicators with different confidence scores feed_data = { "indicators": [ @@ -477,18 +477,18 @@ async def test_indicator_confidence_scoring(self, ioc_manager): {"indicator": "low-conf.com", "type": "domain", "confidence": 30} ] } - + await ioc_manager.update_feed_from_content( "Confidence Test", json.dumps(feed_data), "domain", "json" ) - + # Both should be blocked for now (confidence filtering would be a premium feature) should_block, _ = await ioc_manager.check_domain('high-conf.com') assert should_block is True - + should_block, _ = await ioc_manager.check_domain('low-conf.com') assert should_block is True - + @pytest.mark.asyncio async def test_cleanup_old_indicators(self, ioc_manager): """Test cleanup of old IOC indicators""" @@ -499,16 +499,16 @@ async def test_cleanup_old_indicators(self, ioc_manager): "domain", "txt" ) - + # Run cleanup (with 0 days to remove everything) deleted = await ioc_manager.cleanup_old_indicators(retention_days=0) - + assert deleted >= 0 # Should return count of deleted indicators - + # Verify indicators were cleaned up should_block, _ = await ioc_manager.check_domain('cleanup1.com') assert should_block is False - + @pytest.mark.asyncio async def test_concurrent_checks(self, ioc_manager): """Test concurrent IOC checks""" @@ -519,35 +519,35 @@ async def test_concurrent_checks(self, ioc_manager): "domain", "txt" ) - + # Perform concurrent checks domains = ['concurrent1.com', 'concurrent2.com', 'concurrent3.com', 'clean.com'] tasks = [ioc_manager.check_domain(domain) for domain in domains] - + results = await asyncio.gather(*tasks) - + assert len(results) == 4 # First 3 should be blocked assert results[0][0] is True # concurrent1.com - assert results[1][0] is True # concurrent2.com + assert results[1][0] is True # concurrent2.com assert results[2][0] is True # concurrent3.com assert results[3][0] is False # clean.com - + @pytest.mark.asyncio async def test_default_feeds_initialization(self, ioc_manager): """Test initialization of default IOC feeds""" # Initialize default feeds await ioc_manager.initialize_default_feeds() - + stats = await ioc_manager.get_stats() - + # Should have registered the default 5 feeds assert stats['feeds']['total'] >= 5 - + # Check that known feed names exist # (This would require checking the actual feed names from the implementation) assert stats['feeds']['enabled'] >= 5 - + def test_domain_normalization(self, ioc_manager): """Test domain name normalization""" # Test various domain formats @@ -557,11 +557,11 @@ def test_domain_normalization(self, ioc_manager): ('example.com.', 'example.com'), ('*.EXAMPLE.COM', '*.example.com') ] - + for input_domain, expected in test_cases: normalized = ioc_manager._normalize_domain(input_domain) assert normalized == expected, f"Failed for {input_domain}" - + def test_ip_normalization(self, ioc_manager): """Test IP address normalization""" test_cases = [ @@ -569,7 +569,7 @@ def test_ip_normalization(self, ioc_manager): ('192.168.1.0/24', '192.168.1.0/24'), ('2001:DB8::1', '2001:db8::1') # IPv6 lowercase ] - + for input_ip, expected in test_cases: normalized = ioc_manager._normalize_ip(input_ip) - assert normalized == expected, f"Failed for {input_ip}" \ No newline at end of file + assert normalized == expected, f"Failed for {input_ip}" diff --git a/dns-server/tests_full_future/test_prometheus_metrics.py b/dns-server/tests_full_future/test_prometheus_metrics.py index b70e0bb2..716a9e15 100644 --- a/dns-server/tests_full_future/test_prometheus_metrics.py +++ b/dns-server/tests_full_future/test_prometheus_metrics.py @@ -12,13 +12,13 @@ from prometheus_metrics import PrometheusMetrics, MetricsCollector, init_prometheus_metrics, get_metrics_instance class TestPrometheusMetrics: - + @pytest.fixture def prometheus_metrics(self, temp_db): """Create Prometheus metrics instance with test database""" db_url = f"sqlite://{temp_db._uri[9:]}" return PrometheusMetrics(db_url) - + def test_metrics_initialization(self, prometheus_metrics): """Test that all metrics are properly initialized""" assert prometheus_metrics.dns_queries_total is not None @@ -38,7 +38,7 @@ def test_metrics_initialization(self, prometheus_metrics): assert prometheus_metrics.dns_memory_usage_bytes is not None assert prometheus_metrics.dns_open_files is not None assert prometheus_metrics.dns_server_info is not None - + def test_record_query_basic(self, prometheus_metrics): """Test recording basic DNS query""" prometheus_metrics.record_query( @@ -48,12 +48,12 @@ def test_record_query_basic(self, prometheus_metrics): response_time=0.05, cache_hit=False ) - + # Check internal stats were updated assert prometheus_metrics.query_stats['A_success'] == 1 assert len(prometheus_metrics.response_times) == 1 assert prometheus_metrics.top_domains['example.com'] == 1 - + def test_record_query_with_cache_hit(self, prometheus_metrics): """Test recording query with cache hit""" prometheus_metrics.record_query( @@ -63,11 +63,11 @@ def test_record_query_with_cache_hit(self, prometheus_metrics): response_time=0.001, cache_hit=True ) - + # Verify cache hit was recorded assert len(prometheus_metrics.response_times) == 1 assert prometheus_metrics.response_times[0] == 0.001 - + def test_record_query_blocked(self, prometheus_metrics): """Test recording blocked query""" prometheus_metrics.record_query( @@ -79,14 +79,14 @@ def test_record_query_blocked(self, prometheus_metrics): blocked=True, block_reason='threat_intelligence' ) - + assert prometheus_metrics.query_stats['A_blocked'] == 1 assert prometheus_metrics.error_counts['blocked'] == 1 - + def test_record_query_with_user_token(self, prometheus_metrics): """Test recording query with user token hash""" token_hash = 'abcd1234567890' - + prometheus_metrics.record_query( domain='user.example.com', record_type='A', @@ -95,37 +95,37 @@ def test_record_query_with_user_token(self, prometheus_metrics): cache_hit=False, token_hash=token_hash ) - + # User metrics should be updated assert len(prometheus_metrics.response_times) == 1 - + def test_record_authentication_failure(self, prometheus_metrics): """Test recording authentication failure""" prometheus_metrics.record_authentication_failure('invalid_token') prometheus_metrics.record_authentication_failure('expired_token') prometheus_metrics.record_authentication_failure('invalid_token') # Duplicate - + # Metrics should be incremented # (We can't easily test Prometheus counter values directly) assert True # Placeholder - in real tests we'd check the counter - + def test_record_upstream_query(self, prometheus_metrics): """Test recording upstream DNS query timing""" prometheus_metrics.record_upstream_query('8.8.8.8', 0.15) prometheus_metrics.record_upstream_query('1.1.1.1', 0.08) prometheus_metrics.record_upstream_query('8.8.8.8', 0.12) - + # Upstream metrics should be recorded # (We can't easily test histogram values directly) assert True # Placeholder - + def test_update_cache_stats(self, prometheus_metrics): """Test updating cache statistics""" prometheus_metrics.update_cache_stats(total_entries=1500, hit_rate=0.85) - + assert prometheus_metrics.cache_hit_rate == 0.85 # Gauges should be updated (not easily testable directly) - + def test_update_ioc_stats(self, prometheus_metrics): """Test updating IOC statistics""" ioc_stats = { @@ -137,117 +137,117 @@ def test_update_ioc_stats(self, prometheus_metrics): ] } } - + prometheus_metrics.update_ioc_stats(ioc_stats) - + # IOC metrics should be updated assert True # Placeholder - would test gauge values - + def test_update_server_health(self, prometheus_metrics): """Test updating server health status""" # Test healthy prometheus_metrics.update_server_health(True) - # Test unhealthy + # Test unhealthy prometheus_metrics.update_server_health(False) # Back to healthy prometheus_metrics.update_server_health(True) - + # Health gauge should be updated assert True # Placeholder - + @patch('psutil.Process') def test_update_system_metrics(self, mock_process, prometheus_metrics): """Test updating system resource metrics""" # Mock psutil mock_memory_info = Mock() mock_memory_info.rss = 100 * 1024 * 1024 # 100MB - + mock_process_instance = Mock() mock_process_instance.memory_info.return_value = mock_memory_info mock_process_instance.open_files.return_value = ['file1', 'file2', 'file3'] mock_process.return_value = mock_process_instance - + prometheus_metrics.update_system_metrics() - + # Should have called psutil methods mock_process_instance.memory_info.assert_called_once() mock_process_instance.open_files.assert_called_once() - + @patch('psutil.Process') def test_update_system_metrics_access_denied(self, mock_process, prometheus_metrics): """Test handling of psutil access denied errors""" import psutil - + mock_process_instance = Mock() mock_process_instance.memory_info.side_effect = psutil.AccessDenied() mock_process_instance.open_files.side_effect = psutil.AccessDenied() mock_process.return_value = mock_process_instance - + # Should not raise exception prometheus_metrics.update_system_metrics() - + # Should have attempted to call psutil methods mock_process_instance.memory_info.assert_called_once() - + def test_update_system_metrics_no_psutil(self, prometheus_metrics): """Test system metrics update when psutil is not available""" with patch.dict('sys.modules', {'psutil': None}): # Should not raise exception prometheus_metrics.update_system_metrics() - + def test_update_top_domains(self, prometheus_metrics): """Test updating top domains metrics""" # Add some domain queries domains = [ ('example.com', 100), ('google.com', 80), - ('github.com', 60), + ('github.com', 60), ('stackoverflow.com', 40), ('amazon.com', 20) ] - + for domain, count in domains: prometheus_metrics.top_domains[domain] = count - + prometheus_metrics.update_top_domains(limit=3) - + # Top domains metric should be updated # (Can't easily test Prometheus gauge values directly) assert True # Placeholder - + def test_get_current_stats(self, prometheus_metrics): """Test getting current statistics summary""" # Add some test data prometheus_metrics.record_query('test1.com', 'A', 'success', 0.05, False) prometheus_metrics.record_query('test2.com', 'A', 'success', 0.03, True) prometheus_metrics.record_query('test3.com', 'A', 'error', 0.10, False) - + prometheus_metrics.cache_hit_rate = 0.75 - + stats = prometheus_metrics.get_current_stats() - + assert 'total_queries' in stats assert 'average_response_time_ms' in stats assert 'cache_hit_rate' in stats assert 'error_rate' in stats assert 'top_domains' in stats - + assert stats['total_queries'] == 3 assert stats['cache_hit_rate'] == 0.75 assert stats['error_rate'] > 0 # Should have some errors assert len(stats['top_domains']) <= 10 - + def test_reset_periodic_stats(self, prometheus_metrics): """Test resetting periodic statistics""" # Add some data prometheus_metrics.response_times.extend([0.1, 0.2, 0.3]) - + assert len(prometheus_metrics.response_times) == 3 - + prometheus_metrics.reset_periodic_stats() - + assert len(prometheus_metrics.response_times) == 0 - + def test_get_user_type_from_token(self, prometheus_metrics): """Test determining user type from token hash""" test_cases = [ @@ -257,86 +257,86 @@ def test_get_user_type_from_token(self, prometheus_metrics): ('ENTERPRISE_ABC', 'enterprise'), # Case insensitive ('normal_token', 'community') ] - + for token_hash, expected_type in test_cases: user_type = prometheus_metrics._get_user_type_from_token(token_hash) assert user_type == expected_type - + def test_get_metrics_endpoint(self, prometheus_metrics): """Test generating Prometheus metrics endpoint""" # Add some test data prometheus_metrics.record_query('endpoint-test.com', 'A', 'success', 0.05, False) prometheus_metrics.update_server_health(True) - + metrics_output, content_type = prometheus_metrics.get_metrics_endpoint() - + assert isinstance(metrics_output, bytes) assert content_type == 'text/plain; version=0.0.4; charset=utf-8' - + # Should contain Prometheus format metrics metrics_str = metrics_output.decode('utf-8') assert 'squawk_dns_queries_total' in metrics_str assert 'squawk_dns_server_health' in metrics_str - + @pytest.mark.asyncio async def test_collect_database_stats(self, prometheus_metrics, temp_db): """Test collecting statistics from database""" # This requires the database to have tables set up await prometheus_metrics.collect_database_stats() - + # Should not raise exceptions assert prometheus_metrics.last_stats_update > 0 - + @pytest.mark.asyncio async def test_collect_database_stats_no_tables(self, prometheus_metrics): """Test database stats collection with missing tables""" # Use in-memory database with no tables prometheus_metrics.db_url = "sqlite:///:memory:" - + await prometheus_metrics.collect_database_stats() - + # Should handle missing tables gracefully assert True # Should not raise exception - + @pytest.mark.asyncio async def test_collect_database_stats_rate_limiting(self, prometheus_metrics): """Test that database stats collection respects rate limiting""" # First call await prometheus_metrics.collect_database_stats() first_update_time = prometheus_metrics.last_stats_update - + # Immediate second call should be skipped await prometheus_metrics.collect_database_stats() second_update_time = prometheus_metrics.last_stats_update - + assert second_update_time == first_update_time # Should be same (skipped) - + def test_concurrent_record_query(self, prometheus_metrics): """Test concurrent query recording""" import threading - + def record_queries(start_domain_num): for i in range(100): prometheus_metrics.record_query( f'concurrent{start_domain_num}-{i}.com', 'A', 'success', 0.01, False ) - + # Start multiple threads threads = [] for i in range(5): thread = threading.Thread(target=record_queries, args=(i,)) threads.append(thread) thread.start() - + # Wait for all threads for thread in threads: thread.join() - + # Should have recorded all queries without errors stats = prometheus_metrics.get_current_stats() assert stats['total_queries'] == 500 # 5 threads * 100 queries each - + def test_domain_name_sanitization(self, prometheus_metrics): """Test domain name sanitization for metrics labels""" test_domains = [ @@ -345,168 +345,168 @@ def test_domain_name_sanitization(self, prometheus_metrics): 'very-long-domain-name-that-should-be-truncated-at-fifty-characters.com', 'UPPERCASE.COM' ] - + for domain in test_domains: prometheus_metrics.top_domains[domain] = 10 - + prometheus_metrics.update_top_domains() - + # Should handle all domains without errors assert True # Placeholder - would check sanitized labels class TestMetricsCollector: - + @pytest.fixture def metrics_collector(self, temp_db): """Create metrics collector with test prometheus instance""" db_url = f"sqlite://{temp_db._uri[9:]}" prometheus_metrics = PrometheusMetrics(db_url) return MetricsCollector(prometheus_metrics, collection_interval=0.1) # Fast interval for testing - + def test_collector_initialization(self, metrics_collector): """Test metrics collector initialization""" assert metrics_collector.metrics is not None assert metrics_collector.collection_interval == 0.1 assert metrics_collector.running is False assert metrics_collector.thread is None - + def test_start_stop_collector(self, metrics_collector): """Test starting and stopping metrics collection""" assert metrics_collector.running is False - + # Start collector metrics_collector.start() assert metrics_collector.running is True assert metrics_collector.thread is not None assert metrics_collector.thread.is_alive() - + # Let it run briefly time.sleep(0.3) - + # Stop collector metrics_collector.stop() assert metrics_collector.running is False - + # Thread should finish time.sleep(0.2) assert not metrics_collector.thread.is_alive() - + def test_collector_double_start(self, metrics_collector): """Test that starting collector twice doesn't create multiple threads""" metrics_collector.start() first_thread = metrics_collector.thread - + # Start again metrics_collector.start() second_thread = metrics_collector.thread - + # Should be same thread assert first_thread == second_thread - + metrics_collector.stop() - + def test_collection_loop_error_handling(self, metrics_collector): """Test that collection loop handles errors gracefully""" # Mock the metrics collection to raise an error with patch.object(metrics_collector.metrics, 'collect_database_stats') as mock_collect: mock_collect.side_effect = Exception("Database error") - + # Start collector metrics_collector.start() time.sleep(0.3) # Let it run and hit the error metrics_collector.stop() - + # Should have attempted collection despite errors assert mock_collect.call_count > 0 class TestGlobalMetricsFunctions: - + def test_init_prometheus_metrics(self, temp_db): """Test global metrics initialization""" db_url = f"sqlite://{temp_db._uri[9:]}" - + metrics = init_prometheus_metrics(db_url, enable_collection=False) - + assert metrics is not None assert isinstance(metrics, PrometheusMetrics) - + # Should be accessible via get_metrics_instance global_metrics = get_metrics_instance() assert global_metrics == metrics - + def test_init_prometheus_metrics_with_collection(self, temp_db): """Test metrics initialization with background collection""" db_url = f"sqlite://{temp_db._uri[9:]}" - + with patch('prometheus_metrics.MetricsCollector') as mock_collector_class: mock_collector = Mock() mock_collector_class.return_value = mock_collector - + metrics = init_prometheus_metrics(db_url, enable_collection=True) - + # Should have created and started collector mock_collector_class.assert_called_once_with(metrics) mock_collector.start.assert_called_once() - + def test_get_metrics_instance_none(self): """Test get_metrics_instance when not initialized""" # Reset global instance import prometheus_metrics prometheus_metrics.prometheus_metrics = None - + result = get_metrics_instance() assert result is None - + def test_metrics_thread_safety(self, temp_db): """Test metrics thread safety with concurrent access""" db_url = f"sqlite://{temp_db._uri[9:]}" metrics = PrometheusMetrics(db_url) - + def worker(): for i in range(50): metrics.record_query(f'thread-{threading.current_thread().ident}-{i}.com', 'A', 'success', 0.01, False) metrics.update_server_health(True) time.sleep(0.001) - + threads = [] for i in range(3): thread = threading.Thread(target=worker) threads.append(thread) thread.start() - + for thread in threads: thread.join() - + # Should complete without deadlocks or errors stats = metrics.get_current_stats() assert stats['total_queries'] == 150 # 3 threads * 50 queries each - + def test_error_handling_in_record_query(self, temp_db): """Test error handling within record_query method""" db_url = f"sqlite://{temp_db._uri[9:]}" metrics = PrometheusMetrics(db_url) - + # Mock one of the internal operations to fail with patch.object(metrics.dns_queries_total, 'labels') as mock_labels: mock_labels.side_effect = Exception("Prometheus error") - + # Should not raise exception metrics.record_query('error-test.com', 'A', 'success', 0.05, False) - + # Should have attempted the operation mock_labels.assert_called_once() - + @patch('prometheus_metrics.generate_latest') def test_metrics_endpoint_generation_error(self, mock_generate, temp_db): """Test handling of errors during metrics generation""" db_url = f"sqlite://{temp_db._uri[9:]}" metrics = PrometheusMetrics(db_url) - + # Mock generate_latest to fail mock_generate.side_effect = Exception("Generation error") - + output, content_type = metrics.get_metrics_endpoint() - + assert isinstance(output, str) # Should return error message assert content_type == "text/plain" - assert "Error generating metrics" in output \ No newline at end of file + assert "Error generating metrics" in output diff --git a/dns-server/tests_full_future/test_selective_dns_routing.py b/dns-server/tests_full_future/test_selective_dns_routing.py index 7abf3a7e..50615a20 100644 --- a/dns-server/tests_full_future/test_selective_dns_routing.py +++ b/dns-server/tests_full_future/test_selective_dns_routing.py @@ -10,7 +10,7 @@ from selective_dns_routing import SelectiveDNSRouter class TestSelectiveDNSRouter: - + @pytest.fixture def dns_router(self, temp_db): """Create selective DNS router instance with test database. @@ -30,7 +30,7 @@ def dns_router(self, temp_db): finally: seed.close() return SelectiveDNSRouter(db_url) - + def test_initialization(self, dns_router): """Test DNS router initialization""" assert dns_router.db_url is not None @@ -39,7 +39,7 @@ def test_initialization(self, dns_router): assert 'internal' in dns_router.zone_visibility_levels assert 'restricted' in dns_router.zone_visibility_levels assert 'private' in dns_router.zone_visibility_levels - + def test_create_user_group(self, dns_router): """Test creating user groups""" result = dns_router.create_group( @@ -47,30 +47,30 @@ def test_create_user_group(self, dns_router): "Engineering team", ["internal", "public"] ) - + assert result['success'] is True assert 'group_id' in result - + # Test duplicate group duplicate_result = dns_router.create_group("engineering", "Duplicate", ["public"]) assert duplicate_result['success'] is False assert 'already exists' in duplicate_result['error'] - + def test_assign_user_to_group(self, dns_router): """Test assigning users to groups""" # Create group first group_result = dns_router.create_group("test-group", "Test Group", ["internal", "public"]) group_id = group_result['group_id'] - + # Assign user to group result = dns_router.assign_user_to_group(1, group_id, "admin") - + assert result['success'] is True - + # Test assigning same user again (should update) result2 = dns_router.assign_user_to_group(1, group_id, "admin") assert result2['success'] is True - + def test_create_dns_zone(self, dns_router): """Test creating DNS zones with visibility levels""" zones = [ @@ -78,306 +78,306 @@ def test_create_dns_zone(self, dns_router): ("internal.company.com", "internal", "Internal company zone"), ("secret.company.com", "private", "Private zone") ] - + for zone_name, visibility, description in zones: result = dns_router.create_dns_zone( zone_name, visibility, description, "admin" ) - + assert result['success'] is True assert 'zone_id' in result - + def test_can_resolve_domain_public(self, dns_router): """Test that all users can resolve public domains""" # Create public zone zone_result = dns_router.create_dns_zone("public.example.com", "public", "Public zone", "admin") - + # Test with any token ID can_resolve = dns_router.can_resolve_domain("test-token-123", "subdomain.public.example.com") assert can_resolve is True - + # Test with no token can_resolve_anon = dns_router.can_resolve_domain(None, "subdomain.public.example.com") assert can_resolve_anon is True - + def test_can_resolve_domain_internal_authorized(self, dns_router): """Test that authorized users can resolve internal domains""" # Create group with internal access group_result = dns_router.create_group("internal-users", "Internal Users", ["internal", "public"]) group_id = group_result['group_id'] - + # Assign user to group dns_router.assign_user_to_group(1, group_id, "admin") - + # Create internal zone dns_router.create_dns_zone("internal.company.com", "internal", "Internal zone", "admin") - + # User should be able to resolve can_resolve = dns_router.can_resolve_domain("test-token-123", "app.internal.company.com") assert can_resolve is True - + def test_can_resolve_domain_internal_unauthorized(self, dns_router): """Test that unauthorized users cannot resolve internal domains""" # Create internal zone dns_router.create_dns_zone("internal.company.com", "internal", "Internal zone", "admin") - + # Create group without internal access group_result = dns_router.create_group("public-only", "Public Only", ["public"]) group_id = group_result['group_id'] dns_router.assign_user_to_group(1, group_id, "admin") - + # User should not be able to resolve can_resolve = dns_router.can_resolve_domain("test-token-123", "app.internal.company.com") assert can_resolve is False - + def test_can_resolve_domain_private_authorized(self, dns_router): """Test that only specifically authorized users can resolve private domains""" # Create private zone zone_result = dns_router.create_dns_zone("private.company.com", "private", "Private zone", "admin") zone_id = zone_result['zone_id'] - + # Create group with private access group_result = dns_router.create_group("executives", "Executives", ["private", "restricted", "internal", "public"]) group_id = group_result['group_id'] - + # Assign user to group dns_router.assign_user_to_group(1, group_id, "admin") - + # Grant specific zone access dns_router.grant_zone_access_to_group(zone_id, group_id, "admin") - + # User should be able to resolve can_resolve = dns_router.can_resolve_domain("test-token-123", "secret.private.company.com") assert can_resolve is True - + def test_can_resolve_domain_private_unauthorized(self, dns_router): """Test that unauthorized users cannot resolve private domains""" # Create private zone dns_router.create_dns_zone("private.company.com", "private", "Private zone", "admin") - + # Create group without private access group_result = dns_router.create_group("regular-users", "Regular Users", ["internal", "public"]) group_id = group_result['group_id'] dns_router.assign_user_to_group(1, group_id, "admin") - + # User should not be able to resolve can_resolve = dns_router.can_resolve_domain("test-token-123", "secret.private.company.com") assert can_resolve is False - + def test_filter_dns_response_allowed(self, dns_router, sample_dns_response): """Test DNS response filtering for allowed domain""" # Create public zone dns_router.create_dns_zone("example.com", "public", "Public zone", "admin") - + filtered_response = dns_router.filter_dns_response( "test-token-123", "example.com", sample_dns_response ) - + # Response should be unchanged assert filtered_response == sample_dns_response assert filtered_response['Status'] == 0 assert len(filtered_response['Answer']) > 0 - + def test_filter_dns_response_blocked(self, dns_router, sample_dns_response): """Test DNS response filtering for blocked domain""" # Create private zone without granting access dns_router.create_dns_zone("secret.example.com", "private", "Private zone", "admin") - + # Create group without private access group_result = dns_router.create_group("limited-users", "Limited Users", ["public"]) dns_router.assign_user_to_group(1, group_result['group_id'], "admin") - + original_response = sample_dns_response.copy() original_response['Answer'][0]['name'] = "secret.example.com" - + filtered_response = dns_router.filter_dns_response( "test-token-123", "secret.example.com", original_response ) - + # Response should be filtered (NXDOMAIN) assert filtered_response['Status'] == 3 # NXDOMAIN assert len(filtered_response['Answer']) == 0 assert filtered_response['Comment'] == "Domain not found" - + def test_wildcard_domain_matching(self, dns_router): """Test wildcard domain matching""" # Create zone with wildcard dns_router.create_dns_zone("*.internal.company.com", "internal", "Internal wildcard", "admin") - + # Create group with internal access group_result = dns_router.create_group("internal-group", "Internal", ["internal", "public"]) dns_router.assign_user_to_group(1, group_result['group_id'], "admin") - + # Test various subdomains test_domains = [ "app.internal.company.com", - "api.internal.company.com", + "api.internal.company.com", "db.internal.company.com", "test.app.internal.company.com" ] - + for domain in test_domains: can_resolve = dns_router.can_resolve_domain("test-token-123", domain) assert can_resolve is True, f"Failed to resolve wildcard domain: {domain}" - + def test_get_user_groups(self, dns_router): """Test getting user's group memberships""" # Create multiple groups group1_result = dns_router.create_group("group1", "Group 1", ["public"]) group2_result = dns_router.create_group("group2", "Group 2", ["internal", "public"]) - + # Assign user to both groups dns_router.assign_user_to_group(1, group1_result['group_id'], "admin") dns_router.assign_user_to_group(1, group2_result['group_id'], "admin") - + # Get user groups groups = dns_router.get_user_groups(1) - + assert len(groups) == 2 group_names = [g['name'] for g in groups] assert "group1" in group_names assert "group2" in group_names - + def test_get_zone_access_levels(self, dns_router): """Test getting access levels for a domain""" # Create zones with different visibility dns_router.create_dns_zone("public.example.com", "public", "Public", "admin") dns_router.create_dns_zone("internal.example.com", "internal", "Internal", "admin") - + # Test access levels public_level = dns_router.get_zone_access_level("public.example.com") assert public_level == "public" - + internal_level = dns_router.get_zone_access_level("internal.example.com") assert internal_level == "internal" - + unknown_level = dns_router.get_zone_access_level("unknown.example.com") assert unknown_level == "public" # Default fallback - + def test_grant_revoke_zone_access(self, dns_router): """Test granting and revoking zone access""" # Create private zone and group zone_result = dns_router.create_dns_zone("private.example.com", "private", "Private", "admin") zone_id = zone_result['zone_id'] - + group_result = dns_router.create_group("special-group", "Special", ["public"]) group_id = group_result['group_id'] - + dns_router.assign_user_to_group(1, group_id, "admin") - + # Initially should not have access can_resolve = dns_router.can_resolve_domain("test-token-123", "private.example.com") assert can_resolve is False - + # Grant access grant_result = dns_router.grant_zone_access_to_group(zone_id, group_id, "admin") assert grant_result['success'] is True - + # Now should have access can_resolve = dns_router.can_resolve_domain("test-token-123", "private.example.com") assert can_resolve is True - + # Revoke access revoke_result = dns_router.revoke_zone_access_from_group(zone_id, group_id, "admin") assert revoke_result['success'] is True - + # Should not have access again can_resolve = dns_router.can_resolve_domain("test-token-123", "private.example.com") assert can_resolve is False - + def test_remove_user_from_group(self, dns_router): """Test removing user from group""" # Create group and assign user group_result = dns_router.create_group("temp-group", "Temporary", ["internal", "public"]) group_id = group_result['group_id'] - + dns_router.assign_user_to_group(1, group_id, "admin") - + # Create internal zone dns_router.create_dns_zone("internal.example.com", "internal", "Internal", "admin") - + # User should have access can_resolve = dns_router.can_resolve_domain("test-token-123", "internal.example.com") assert can_resolve is True - + # Remove user from group remove_result = dns_router.remove_user_from_group(1, group_id, "admin") assert remove_result['success'] is True - + # User should no longer have access can_resolve = dns_router.can_resolve_domain("test-token-123", "internal.example.com") assert can_resolve is False - + def test_get_routing_stats(self, dns_router): """Test getting routing statistics""" # Create some test data dns_router.create_group("stats-group1", "Stats Group 1", ["public"]) dns_router.create_group("stats-group2", "Stats Group 2", ["internal", "public"]) - + dns_router.create_dns_zone("stats.example.com", "public", "Stats zone", "admin") dns_router.create_dns_zone("internal-stats.example.com", "internal", "Internal stats", "admin") - + dns_router.assign_user_to_group(1, 1, "admin") # Assuming group IDs start at 1 dns_router.assign_user_to_group(2, 2, "admin") - + stats = dns_router.get_routing_stats() - + assert 'groups' in stats - assert 'zones' in stats + assert 'zones' in stats assert 'user_assignments' in stats assert 'zone_access_grants' in stats - + assert stats['groups']['total'] >= 2 assert stats['zones']['total'] >= 2 assert stats['user_assignments']['total'] >= 2 - + def test_domain_hierarchy_matching(self, dns_router): """Test domain hierarchy matching for zones""" # Create hierarchical zones dns_router.create_dns_zone("company.com", "public", "Company root", "admin") dns_router.create_dns_zone("internal.company.com", "internal", "Internal subdomain", "admin") dns_router.create_dns_zone("secret.internal.company.com", "private", "Secret subdomain", "admin") - + # Create user with only internal access group_result = dns_router.create_group("internal-only", "Internal Only", ["internal", "public"]) dns_router.assign_user_to_group(1, group_result['group_id'], "admin") - + # Test domain resolution assert dns_router.can_resolve_domain("test-token-123", "company.com") is True # Public assert dns_router.can_resolve_domain("test-token-123", "www.company.com") is True # Public assert dns_router.can_resolve_domain("test-token-123", "app.internal.company.com") is True # Internal assert dns_router.can_resolve_domain("test-token-123", "api.secret.internal.company.com") is False # Private - + def test_token_to_user_id_mapping(self, dns_router, sample_token_data): """Test mapping tokens to user IDs""" # This tests the internal _get_user_id_from_token method user_id = dns_router._get_user_id_from_token(sample_token_data['token']) - + assert user_id is not None assert user_id == sample_token_data['token_id'] - + # Test with invalid token invalid_user_id = dns_router._get_user_id_from_token("invalid-token-123") assert invalid_user_id is None - + def test_caching_behavior(self, dns_router): """Test caching of DNS routing decisions""" # Create zone and group dns_router.create_dns_zone("cached.example.com", "internal", "Cached zone", "admin") group_result = dns_router.create_group("cached-group", "Cached", ["internal", "public"]) dns_router.assign_user_to_group(1, group_result['group_id'], "admin") - + # First resolution (should cache) can_resolve1 = dns_router.can_resolve_domain("test-token-123", "cached.example.com") assert can_resolve1 is True - + # Second resolution (should use cache if implemented) can_resolve2 = dns_router.can_resolve_domain("test-token-123", "cached.example.com") assert can_resolve2 is True - + # Results should be consistent assert can_resolve1 == can_resolve2 - + def test_bulk_operations(self, dns_router): """Test bulk user and zone operations""" # Bulk create groups @@ -386,112 +386,112 @@ def test_bulk_operations(self, dns_router): ("bulk-group-2", "Bulk Group 2", ["internal", "public"]), ("bulk-group-3", "Bulk Group 3", ["restricted", "internal", "public"]) ] - + group_ids = [] for name, desc, levels in groups_data: result = dns_router.create_group(name, desc, levels) assert result['success'] is True group_ids.append(result['group_id']) - + # Bulk create zones zones_data = [ ("bulk1.example.com", "public"), ("bulk2.example.com", "internal"), ("bulk3.example.com", "restricted") ] - + for zone_name, visibility in zones_data: result = dns_router.create_dns_zone(zone_name, visibility, f"Bulk zone {zone_name}", "admin") assert result['success'] is True - + # Bulk assign users for i, group_id in enumerate(group_ids): result = dns_router.assign_user_to_group(i + 10, group_id, "admin") # User IDs 10, 11, 12 assert result['success'] is True - + def test_edge_cases(self, dns_router): """Test edge cases and error conditions""" # Test with empty domain can_resolve = dns_router.can_resolve_domain("test-token-123", "") assert can_resolve is False - + # Test with None domain can_resolve = dns_router.can_resolve_domain("test-token-123", None) assert can_resolve is False - + # Test with malformed domain can_resolve = dns_router.can_resolve_domain("test-token-123", "invalid..domain") assert can_resolve is False - + # Test with very long domain long_domain = "a" * 300 + ".example.com" can_resolve = dns_router.can_resolve_domain("test-token-123", long_domain) assert can_resolve in [True, False] # Should handle gracefully - + def test_concurrent_access(self, dns_router): """Test concurrent access to routing functions""" import threading - + # Create test data dns_router.create_group("concurrent-group", "Concurrent", ["internal", "public"]) dns_router.create_dns_zone("concurrent.example.com", "internal", "Concurrent zone", "admin") dns_router.assign_user_to_group(1, 1, "admin") - + results = [] - + def test_resolution(): for i in range(10): result = dns_router.can_resolve_domain("test-token-123", f"test{i}.concurrent.example.com") results.append(result) - + # Run concurrent tests threads = [] for i in range(3): thread = threading.Thread(target=test_resolution) threads.append(thread) thread.start() - + for thread in threads: thread.join() - + # All results should be consistent assert len(results) == 30 # 3 threads * 10 results each assert all(r in [True, False] for r in results) # All should be valid boolean results - + def test_delete_group(self, dns_router): """Test deleting user groups""" # Create group group_result = dns_router.create_group("delete-me", "Delete Me", ["public"]) group_id = group_result['group_id'] - + # Assign user to group dns_router.assign_user_to_group(1, group_id, "admin") - + # Delete group delete_result = dns_router.delete_group(group_id, "admin") assert delete_result['success'] is True - + # User should no longer be in any groups related to this user_groups = dns_router.get_user_groups(1) group_names = [g['name'] for g in user_groups] assert "delete-me" not in group_names - + def test_zone_inheritance(self, dns_router): """Test zone inheritance behavior""" # Create parent zone dns_router.create_dns_zone("parent.example.com", "internal", "Parent zone", "admin") - + # Create group with internal access group_result = dns_router.create_group("inherit-group", "Inherit", ["internal", "public"]) dns_router.assign_user_to_group(1, group_result['group_id'], "admin") - + # Test that subdomains inherit parent zone access test_subdomains = [ "child.parent.example.com", "grandchild.child.parent.example.com", "api.v1.parent.example.com" ] - + for subdomain in test_subdomains: can_resolve = dns_router.can_resolve_domain("test-token-123", subdomain) - assert can_resolve is True, f"Subdomain inheritance failed for: {subdomain}" \ No newline at end of file + assert can_resolve is True, f"Subdomain inheritance failed for: {subdomain}" diff --git a/dns-server/tests_full_future/test_whois_manager.py b/dns-server/tests_full_future/test_whois_manager.py index 3f134e91..144a81cf 100644 --- a/dns-server/tests_full_future/test_whois_manager.py +++ b/dns-server/tests_full_future/test_whois_manager.py @@ -11,13 +11,13 @@ from whois_manager import WHOISManager class TestWHOISManager: - + @pytest.fixture def whois_manager(self, temp_db): """Create WHOIS manager instance with test database""" db_url = f"sqlite://{temp_db._uri[9:]}" # Extract path from DAL URI return WHOISManager(db_url) - + @pytest.mark.asyncio async def test_lookup_domain_success(self, whois_manager, mock_whois_response): """Test successful domain WHOIS lookup""" @@ -32,16 +32,16 @@ async def test_lookup_domain_success(self, whois_manager, mock_whois_response): 'status': ['clientTransferProhibited'], 'emails': ['admin@example.com'] } - + result = await whois_manager.lookup_domain('example.com', '127.0.0.1') - + assert result['success'] is True assert result['domain'] == 'example.com' assert result['registrar'] == 'Example Registrar Inc.' assert result['query_type'] == 'domain' assert 'cached' in result mock_whois.assert_called_once_with('example.com') - + @pytest.mark.asyncio async def test_lookup_domain_cached(self, whois_manager): """Test domain lookup returns cached result""" @@ -51,18 +51,18 @@ async def test_lookup_domain_cached(self, whois_manager): 'domain_name': 'cached.example.com', 'registrar': 'Test Registrar' } - + result1 = await whois_manager.lookup_domain('cached.example.com', '127.0.0.1') assert result1['cached'] is False - + # Second lookup should be cached result2 = await whois_manager.lookup_domain('cached.example.com', '127.0.0.1') assert result2['cached'] is True assert result2['registrar'] == 'Test Registrar' - + # WHOIS should only be called once mock_whois.assert_called_once() - + @pytest.mark.asyncio async def test_lookup_domain_force_refresh(self, whois_manager): """Test force refresh bypasses cache""" @@ -71,29 +71,29 @@ async def test_lookup_domain_force_refresh(self, whois_manager): 'domain_name': 'refresh.example.com', 'registrar': 'Test Registrar' } - + # First lookup await whois_manager.lookup_domain('refresh.example.com', '127.0.0.1') - + # Second lookup with force refresh result = await whois_manager.lookup_domain('refresh.example.com', '127.0.0.1', force_refresh=True) - + assert result['cached'] is False # WHOIS should be called twice assert mock_whois.call_count == 2 - + @pytest.mark.asyncio async def test_lookup_domain_whois_failure(self, whois_manager): """Test handling of WHOIS lookup failures""" with patch('whois.whois') as mock_whois: mock_whois.side_effect = Exception("WHOIS lookup failed") - + result = await whois_manager.lookup_domain('invalid.example.com', '127.0.0.1') - + assert result['success'] is False assert 'error' in result assert result['domain'] == 'invalid.example.com' - + @pytest.mark.asyncio async def test_lookup_ip_success(self, whois_manager): """Test successful IP WHOIS lookup""" @@ -110,15 +110,15 @@ async def test_lookup_ip_success(self, whois_manager): 'remarks': [{'description': ['Test network']}] } mock_ipwhois.return_value = mock_instance - + result = await whois_manager.lookup_ip('192.0.2.100', '127.0.0.1') - + assert result['success'] is True assert result['ip'] == '192.0.2.100' assert result['query_type'] == 'ip' assert result['network_name'] == 'TEST-NET' assert result['country'] == 'US' - + @pytest.mark.asyncio async def test_lookup_invalid_domain(self, whois_manager, invalid_domains): """Test lookup of invalid domain names""" @@ -126,7 +126,7 @@ async def test_lookup_invalid_domain(self, whois_manager, invalid_domains): result = await whois_manager.lookup_domain(invalid_domain, '127.0.0.1') assert result['success'] is False assert 'invalid' in result['error'].lower() or 'format' in result['error'].lower() - + @pytest.mark.asyncio async def test_search_whois_registrar(self, whois_manager): """Test WHOIS search by registrar""" @@ -137,20 +137,20 @@ async def test_search_whois_registrar(self, whois_manager): 'registrar': 'Example Registrar Inc.' } await whois_manager.lookup_domain('test1.com', '127.0.0.1') - + mock_whois.return_value = { - 'domain_name': 'test2.com', + 'domain_name': 'test2.com', 'registrar': 'Different Registrar LLC' } await whois_manager.lookup_domain('test2.com', '127.0.0.1') - + # Search by registrar results = await whois_manager.search_whois('Example', 'registrar', 10) - + assert len(results) >= 1 found = any('test1.com' in str(result) for result in results) assert found - + @pytest.mark.asyncio async def test_search_whois_organization(self, whois_manager): """Test WHOIS search by organization""" @@ -160,13 +160,13 @@ async def test_search_whois_organization(self, whois_manager): 'org': 'Test Organization Inc.' } await whois_manager.lookup_domain('org-test.com', '127.0.0.1') - + results = await whois_manager.search_whois('Test Organization', 'organization', 10) - + assert len(results) >= 1 found = any('org-test.com' in str(result) for result in results) assert found - + @pytest.mark.asyncio async def test_search_whois_nameserver(self, whois_manager): """Test WHOIS search by nameserver""" @@ -176,13 +176,13 @@ async def test_search_whois_nameserver(self, whois_manager): 'name_servers': ['ns1.example.com', 'ns2.example.com'] } await whois_manager.lookup_domain('ns-test.com', '127.0.0.1') - + results = await whois_manager.search_whois('ns1.example.com', 'nameserver', 10) - + assert len(results) >= 1 found = any('ns-test.com' in str(result) for result in results) assert found - + @pytest.mark.asyncio async def test_search_whois_general(self, whois_manager): """Test general WHOIS search across all fields""" @@ -193,13 +193,13 @@ async def test_search_whois_general(self, whois_manager): 'org': 'Unique Organization' } await whois_manager.lookup_domain('general-test.com', '127.0.0.1') - + results = await whois_manager.search_whois('Unique', None, 10) - + assert len(results) >= 1 found = any('general-test.com' in str(result) for result in results) assert found - + @pytest.mark.asyncio async def test_get_stats(self, whois_manager): """Test WHOIS statistics collection""" @@ -210,7 +210,7 @@ async def test_get_stats(self, whois_manager): 'registrar': 'Stats Registrar' } await whois_manager.lookup_domain('stats-test.com', '127.0.0.1') - + with patch('ipwhois.IPWhois') as mock_ipwhois: mock_instance = Mock() mock_instance.lookup_rdap.return_value = { @@ -218,16 +218,16 @@ async def test_get_stats(self, whois_manager): } mock_ipwhois.return_value = mock_instance await whois_manager.lookup_ip('192.0.2.1', '127.0.0.1') - + stats = await whois_manager.get_stats() - + assert 'queries' in stats assert 'cache' in stats assert stats['queries']['total'] >= 2 assert stats['queries']['domain_queries'] >= 1 assert stats['queries']['ip_queries'] >= 1 assert stats['cache']['total_entries'] >= 2 - + @pytest.mark.asyncio async def test_cleanup_old_data(self, whois_manager): """Test cleanup of old WHOIS data""" @@ -238,35 +238,35 @@ async def test_cleanup_old_data(self, whois_manager): 'registrar': 'Cleanup Registrar' } await whois_manager.lookup_domain('cleanup-test.com', '127.0.0.1') - + # Cleanup with 0 days retention (should remove everything) deleted = await whois_manager.cleanup_old_data(retention_days=0) - + assert deleted >= 0 # Should return number of deleted records - + # Verify data was cleaned up stats = await whois_manager.get_stats() assert stats['cache']['total_entries'] == 0 - + @pytest.mark.asyncio async def test_rate_limiting(self, whois_manager): """Test WHOIS rate limiting functionality""" with patch('whois.whois') as mock_whois: mock_whois.return_value = {'domain_name': 'rate-test.com'} - + # Make multiple rapid requests tasks = [] for i in range(5): task = whois_manager.lookup_domain(f'rate-test-{i}.com', '127.0.0.1') tasks.append(task) - + results = await asyncio.gather(*tasks, return_exceptions=True) - + # All should complete (rate limiting is internal) assert len(results) == 5 successful = sum(1 for r in results if isinstance(r, dict) and r.get('success')) assert successful == 5 - + @pytest.mark.asyncio async def test_concurrent_cache_access(self, whois_manager): """Test concurrent access to WHOIS cache""" @@ -275,44 +275,44 @@ async def test_concurrent_cache_access(self, whois_manager): 'domain_name': 'concurrent-test.com', 'registrar': 'Concurrent Registrar' } - + # Make concurrent requests for same domain tasks = [] for i in range(3): task = whois_manager.lookup_domain('concurrent-test.com', f'127.0.0.{i+1}') tasks.append(task) - + results = await asyncio.gather(*tasks) - + # All should succeed assert all(r['success'] for r in results) assert all(r['domain'] == 'concurrent-test.com' for r in results) - + # Only one should have called the actual WHOIS (others cached) cached_count = sum(1 for r in results if r['cached']) assert cached_count >= 1 # At least one should be cached - + def test_domain_validation(self, whois_manager, valid_domains, invalid_domains): """Test domain name validation""" # Test valid domains for domain in valid_domains: assert whois_manager._is_valid_domain(domain), f"Valid domain failed: {domain}" - + # Test invalid domains for domain in invalid_domains: assert not whois_manager._is_valid_domain(domain), f"Invalid domain passed: {domain}" - + def test_ip_validation(self, whois_manager): """Test IP address validation""" valid_ips = ['192.168.1.1', '8.8.8.8', '2001:db8::1', '::1'] invalid_ips = ['256.256.256.256', 'not.an.ip', '', '192.168.1'] - + for ip in valid_ips: assert whois_manager._is_valid_ip(ip), f"Valid IP failed: {ip}" - + for ip in invalid_ips: assert not whois_manager._is_valid_ip(ip), f"Invalid IP passed: {ip}" - + @pytest.mark.asyncio async def test_whois_data_parsing(self, whois_manager): """Test WHOIS data parsing and normalization""" @@ -325,27 +325,27 @@ async def test_whois_data_parsing(self, whois_manager): 'name_servers': ['NS1.EXAMPLE.COM', 'ns2.example.com'], # Mixed case 'status': ['clientTransferProhibited https://...', 'clientUpdateProhibited'] } - + result = await whois_manager.lookup_domain('example.com', '127.0.0.1') - + assert result['success'] is True assert result['domain'] == 'example.com' # Normalized to lowercase assert isinstance(result['nameservers'], list) assert len(result['nameservers']) == 2 assert all(ns.islower() for ns in result['nameservers']) # Normalized to lowercase - - @pytest.mark.asyncio + + @pytest.mark.asyncio async def test_error_handling_network_timeout(self, whois_manager): """Test handling of network timeouts""" with patch('whois.whois') as mock_whois: import socket mock_whois.side_effect = socket.timeout("Connection timed out") - + result = await whois_manager.lookup_domain('timeout-test.com', '127.0.0.1') - + assert result['success'] is False assert 'timeout' in result['error'].lower() - + @pytest.mark.asyncio async def test_ip_whois_rdap_fallback(self, whois_manager): """Test IP WHOIS RDAP lookup with fallback""" @@ -362,13 +362,13 @@ async def test_ip_whois_rdap_fallback(self, whois_manager): }] } mock_ipwhois.return_value = mock_instance - + result = await whois_manager.lookup_ip('203.0.113.1', '127.0.0.1') - + assert result['success'] is True assert result['network_name'] == 'LEGACY-NET' assert result['country'] == 'US' - + # Should have tried RDAP first, then legacy mock_instance.lookup_rdap.assert_called_once() - mock_instance.lookup_whois.assert_called_once() \ No newline at end of file + mock_instance.lookup_whois.assert_called_once() diff --git a/dns-server/tests_full_future/unittests.py b/dns-server/tests_full_future/unittests.py index d393be46..3242ea82 100644 --- a/dns-server/tests_full_future/unittests.py +++ b/dns-server/tests_full_future/unittests.py @@ -15,4 +15,4 @@ def test_example(self): self.assertEqual(1, 1) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/docker-compose.license.yml b/docker-compose.license.yml index 2bb8b4cc..58b0239e 100644 --- a/docker-compose.license.yml +++ b/docker-compose.license.yml @@ -151,4 +151,4 @@ networks: volumes: license_db_data: - valkey_data: \ No newline at end of file + valkey_data: diff --git a/docker-compose.yml b/docker-compose.yml index 963af2fc..d2df7e6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -290,4 +290,4 @@ volumes: valkey-data: postgres-data: prometheus-data: - grafana-data: \ No newline at end of file + grafana-data: diff --git a/docs/.gitignore b/docs/.gitignore index fec4bbaa..570f2dfb 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -18,4 +18,4 @@ venv/ # OS files .DS_Store -Thumbs.db \ No newline at end of file +Thumbs.db diff --git a/docs/API.md b/docs/API.md index 5d4a2dab..e01ba183 100644 --- a/docs/API.md +++ b/docs/API.md @@ -67,7 +67,7 @@ Authorization: Bearer your-token-here curl -H "Authorization: Bearer abc123def456" \ "https://dns.example.com/dns-query?name=example.com&type=A" -# Management API with admin token +# Management API with admin token curl -H "Authorization: Bearer admin-token-789" \ "http://localhost:8000/dns_console/api/tokens" ``` @@ -98,7 +98,7 @@ GET /dns-query curl -H "Authorization: Bearer TOKEN" \ "https://dns.example.com/dns-query?name=example.com&type=A" -# AAAA record query +# AAAA record query curl -H "Authorization: Bearer TOKEN" \ "https://dns.example.com/dns-query?name=example.com&type=AAAA" @@ -119,7 +119,7 @@ curl -H "Authorization: Bearer TOKEN" \ "Answer": [ { "name": "example.com", - "type": "A", + "type": "A", "data": "93.184.216.34" } ] @@ -260,7 +260,7 @@ GET /dns_console/api/tokens/{id} "description": "Main domain" }, { - "id": 2, + "id": 2, "name": "*.api.example.com", "description": "API subdomains" } @@ -382,7 +382,7 @@ GET /dns_console/api/domains }, { "id": 2, - "name": "*.api.example.com", + "name": "*.api.example.com", "description": "API subdomain wildcard", "created_at": "2024-01-01T00:00:00Z", "token_count": 1 @@ -391,7 +391,7 @@ GET /dns_console/api/domains "id": 3, "name": "*", "description": "Wildcard - all domains", - "created_at": "2024-01-01T00:00:00Z", + "created_at": "2024-01-01T00:00:00Z", "token_count": 1 } ] @@ -458,7 +458,7 @@ PUT /dns_console/api/domains/{id} "success": true, "data": { "id": 4, - "name": "updated-domain.com", + "name": "updated-domain.com", "description": "Updated description" }, "message": "Domain updated successfully" @@ -715,7 +715,7 @@ POST /dns_console/blacklist/domain/add **Form Data:** - `domain`: Domain to block (string, required) -- `reason`: Reason for blocking (string, optional) +- `reason`: Reason for blocking (string, optional) - `added_by`: Administrator name (string, optional) #### Add IP to Blacklist @@ -808,7 +808,7 @@ GET/POST /dns_console/mfa/setup **POST Actions:** - `action=generate`: Generate new MFA secret and QR code -- `action=verify`: Verify MFA token and enable MFA +- `action=verify`: Verify MFA token and enable MFA - `action=disable`: Disable MFA (requires password) #### Verify MFA @@ -897,7 +897,7 @@ POST /dns_console/auth/register **Form Data:** - `email`: Email address (string, required) -- `password`: Password (string, required) +- `password`: Password (string, required) - `first_name`: First name (string, optional) - `last_name`: Last name (string, optional) @@ -1803,7 +1803,7 @@ GET /dns_console/api/stats "queries": 25000 }, { - "domain": "example.com", + "domain": "example.com", "queries": 15000 } ], @@ -1872,7 +1872,7 @@ curl -H "Authorization: Bearer TOKEN" \ "timestamp": "2024-01-03T09:59:58Z", "token_name": "Development", "domain_queried": "test.example.com", - "query_type": "AAAA", + "query_type": "AAAA", "status": "denied", "client_ip": "10.0.1.50", "response_time_ms": 12 @@ -1994,7 +1994,7 @@ All API errors follow a consistent format: { "success": false, "error": { - "code": "VALIDATION_ERROR", + "code": "VALIDATION_ERROR", "message": "Token name is required and must be unique", "details": { "field": "name", @@ -2080,43 +2080,43 @@ class SquawkClient: 'Authorization': f'Bearer {token}', 'Content-Type': 'application/json' }) - + def dns_query(self, domain: str, record_type: str = 'A') -> Dict: """Perform DNS query""" url = f"{self.base_url}/dns-query" params = {'name': domain, 'type': record_type} - + response = self.session.get(url, params=params) response.raise_for_status() return response.json() - + def list_tokens(self) -> List[Dict]: """List all tokens""" url = f"{self.base_url}/dns_console/api/tokens" response = self.session.get(url) response.raise_for_status() return response.json()['data'] - - def create_token(self, name: str, description: str = None, + + def create_token(self, name: str, description: str = None, domains: List[str] = None) -> Dict: """Create new token""" url = f"{self.base_url}/dns_console/api/tokens" payload = {'name': name} - + if description: payload['description'] = description if domains: payload['domains'] = domains - + response = self.session.post(url, json=payload) response.raise_for_status() return response.json()['data'] - + def grant_permission(self, token_id: int, domain_id: int) -> bool: """Grant domain permission to token""" url = f"{self.base_url}/dns_console/api/permissions" payload = {'token_id': token_id, 'domain_id': domain_id} - + response = self.session.post(url, json=payload) response.raise_for_status() return response.json()['success'] @@ -2149,73 +2149,73 @@ class SquawkClient { 'Content-Type': 'application/json' }; } - + async dnsQuery(domain, recordType = 'A') { const url = `${this.baseUrl}/dns-query?name=${domain}&type=${recordType}`; const response = await fetch(url, { headers: this.headers }); - + if (!response.ok) { throw new Error(`DNS query failed: ${response.statusText}`); } - + return await response.json(); } - + async listTokens() { const url = `${this.baseUrl}/dns_console/api/tokens`; const response = await fetch(url, { headers: this.headers }); - + if (!response.ok) { throw new Error(`Failed to list tokens: ${response.statusText}`); } - + const data = await response.json(); return data.data; } - + async createToken(name, description = null, domains = null) { const url = `${this.baseUrl}/dns_console/api/tokens`; const payload = { name }; - + if (description) payload.description = description; if (domains) payload.domains = domains; - + const response = await fetch(url, { method: 'POST', headers: this.headers, body: JSON.stringify(payload) }); - + if (!response.ok) { throw new Error(`Failed to create token: ${response.statusText}`); } - + const data = await response.json(); return data.data; } - + async getQueryLogs(options = {}) { const params = new URLSearchParams(); - + Object.entries(options).forEach(([key, value]) => { if (value !== null && value !== undefined) { params.append(key, value); } }); - + const url = `${this.baseUrl}/dns_console/api/logs?${params}`; const response = await fetch(url, { headers: this.headers }); - + if (!response.ok) { throw new Error(`Failed to get logs: ${response.statusText}`); } - + return await response.json(); } } @@ -2242,8 +2242,8 @@ client.createToken('Web App Token', 'Token for web application', ['example.com'] }); // Get recent logs -client.getQueryLogs({ - per_page: 100, +client.getQueryLogs({ + per_page: 100, status: 'allowed', start_date: '2024-01-01T00:00:00Z' }).then(logs => { @@ -2305,29 +2305,29 @@ func (c *Client) DNSQuery(domain, recordType string) (*DNSResponse, error) { q.Set("name", domain) q.Set("type", recordType) u.RawQuery = q.Encode() - + req, err := http.NewRequest("GET", u.String(), nil) if err != nil { return nil, err } - + req.Header.Set("Authorization", "Bearer "+c.Token) - + resp, err := c.HTTPClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() - + if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("DNS query failed: %s", resp.Status) } - + var result DNSResponse if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return nil, err } - + return &result, nil } @@ -2337,66 +2337,66 @@ func (c *Client) CreateToken(name, description string, domains []string) (*Token "description": description, "domains": domains, } - + jsonPayload, err := json.Marshal(payload) if err != nil { return nil, err } - - req, err := http.NewRequest("POST", c.BaseURL+"/dns_console/api/tokens", + + req, err := http.NewRequest("POST", c.BaseURL+"/dns_console/api/tokens", bytes.NewBuffer(jsonPayload)) if err != nil { return nil, err } - + req.Header.Set("Authorization", "Bearer "+c.Token) req.Header.Set("Content-Type", "application/json") - + resp, err := c.HTTPClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() - + if resp.StatusCode != http.StatusCreated { return nil, fmt.Errorf("create token failed: %s", resp.Status) } - + var result struct { Success bool `json:"success"` Data *Token `json:"data"` } - + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return nil, err } - + return result.Data, nil } // Usage example func main() { client := NewClient("https://dns.example.com", "your-admin-token") - + // DNS query result, err := client.DNSQuery("example.com", "A") if err != nil { panic(err) } - + if len(result.Answer) > 0 { fmt.Printf("IP: %s\n", result.Answer[0].Data) } - + // Create token - token, err := client.CreateToken("Go App Token", "Token for Go application", + token, err := client.CreateToken("Go App Token", "Token for Go application", []string{"api.example.com"}) if err != nil { panic(err) } - + fmt.Printf("New token: %s\n", token.Token) } ``` -This comprehensive API documentation provides all the information needed to integrate with the Squawk DNS system, from basic DNS queries to advanced token and permission management. \ No newline at end of file +This comprehensive API documentation provides all the information needed to integrate with the Squawk DNS system, from basic DNS queries to advanced token and permission management. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 495a5c02..462a0562 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -92,7 +92,7 @@ The DNS server is the core component responsible for handling DNS-over-HTTPS req class DNSHandler(http.server.BaseHTTPRequestHandler): """ Main request handler implementing DoH protocol. - + Responsibilities: - HTTP request parsing - Authentication enforcement @@ -209,7 +209,7 @@ dns_console/ │ └─ DNSForwarder → DNSOverHTTPSClient └─ Direct HTTPS DoH request -2. Server Processing +2. Server Processing ├─ HTTP request parsing ├─ Authorization header extraction ├─ Token validation against database @@ -246,7 +246,7 @@ Web Console Database ←→ DNS Server Database │ ├─ Real-time token validation ├─ Shared database schema - ├─ Transaction consistency + ├─ Transaction consistency └─ Activity logging coordination ``` @@ -263,7 +263,7 @@ Web Console Database ←→ DNS Server Database │ name │ │ created_at │ │ description │ │ description │ └──────────────┘ │ created_at │ │ active │ └─────────────┘ -│ created_at │ +│ created_at │ │ last_used │ ┌──────────────┐ └─────────────┘ │ query_logs │ ├──────────────┤ @@ -376,18 +376,18 @@ def check_domain_permission(token_id: int, domain: str) -> bool: # 1. Check for wildcard permission if has_wildcard_permission(token_id): return True - + # 2. Check direct domain match if has_direct_permission(token_id, domain): return True - + # 3. Check parent domain permissions parts = domain.split('.') for i in range(len(parts)): parent_domain = '.'.join(parts[i:]) if has_direct_permission(token_id, parent_domain): return True - + return False ``` @@ -463,7 +463,7 @@ def secure_token_comparison(provided: str, stored: str) -> bool: "event_type": "dns_query", "token_id": 123, "domain": "example.com", - "query_type": "A", + "query_type": "A", "status": "allowed", "client_ip": "192.168.1.100", "response_time_ms": 150 @@ -485,14 +485,14 @@ def secure_token_comparison(provided: str, stored: str) -> bool: **Database Performance** ```sql -- Optimized token lookup query -SELECT t.active, t.last_used -FROM tokens t +SELECT t.active, t.last_used +FROM tokens t WHERE t.token = ? AND t.active = TRUE LIMIT 1; -- Domain permission check with index usage -SELECT 1 FROM token_domains td -JOIN domains d ON td.domain_id = d.id +SELECT 1 FROM token_domains td +JOIN domains d ON td.domain_id = d.id WHERE td.token_id = ? AND d.name IN (?, ?, ?) LIMIT 1; ``` @@ -514,7 +514,7 @@ class TokenCache: def __init__(self, ttl=300): # 5-minute TTL self.cache = {} self.ttl = ttl - + def get_permissions(self, token: str) -> Optional[List[str]]: """Get cached token permissions""" pass @@ -549,13 +549,13 @@ services: environment: - NODE_ID=1 - DATABASE_URL=postgresql://shared-db - + dns-server-2: - image: squawk:latest + image: squawk:latest environment: - NODE_ID=2 - DATABASE_URL=postgresql://shared-db - + load-balancer: image: nginx:latest ports: @@ -648,7 +648,7 @@ spec: memory: "256Mi" cpu: "200m" requests: - memory: "128Mi" + memory: "128Mi" cpu: "100m" ``` @@ -660,7 +660,7 @@ spec: ```python UPSTREAM_RESOLVERS = [ '8.8.8.8', # Google DNS - '8.8.4.4', # Google DNS Secondary + '8.8.4.4', # Google DNS Secondary '1.1.1.1', # Cloudflare '1.0.0.1', # Cloudflare Secondary '208.67.222.222', # OpenDNS @@ -696,7 +696,7 @@ GET /api/v1/tokens/{id} # Get token details PUT /api/v1/tokens/{id} # Update token DELETE /api/v1/tokens/{id} # Delete token -# Domain management API +# Domain management API GET /api/v1/domains # List domains POST /api/v1/domains # Add domain DELETE /api/v1/domains/{id} # Remove domain @@ -720,7 +720,7 @@ def register_service(): name='squawk-dns', service_id=f'squawk-dns-{NODE_ID}', port=8080, - check=consul.Check.http('http://localhost:8080/health', + check=consul.Check.http('http://localhost:8080/health', interval='10s') ) ``` @@ -735,7 +735,7 @@ class DNSResponseCache: """Distributed DNS response caching""" def __init__(self): self.redis = redis.Redis(host='cache-cluster') - + def get_cached_response(self, query: str, record_type: str): key = f"dns:{query}:{record_type}" return self.redis.get(key) @@ -980,4 +980,4 @@ DELETE /api/v1/dhcp/reservations/{id} # Delete reservation - gRPC internal communication - Protocol buffer serialization -This architecture document provides a comprehensive overview of Squawk's design principles, component interactions, and future evolution path. The system is designed to be secure, scalable, and maintainable while providing excellent performance for DNS resolution services. \ No newline at end of file +This architecture document provides a comprehensive overview of Squawk's design principles, component interactions, and future evolution path. The system is designed to be secure, scalable, and maintainable while providing excellent performance for DNS resolution services. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 52de5731..4a53a182 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -51,7 +51,7 @@ Or visit: https://cla.penguintech.group/squawk ### Prerequisites -- Python 3.8+ +- Python 3.8+ - Git - Docker (optional, for testing) - Basic understanding of DNS protocols @@ -102,8 +102,7 @@ export TEST_DB_URL="sqlite:///test.db" # requirements-dev.txt pytest>=7.0.0 pytest-cov>=4.0.0 -black>=22.0.0 -flake8>=5.0.0 +ruff>=0.8.4 mypy>=0.991 pre-commit>=2.20.0 pytest-mock>=3.8.0 @@ -113,20 +112,17 @@ factory-boy>=3.2.0 ### Code Formatting -We use several tools for code quality: +We use several tools for code quality (ruff supersedes flake8/black/isort — see `pyproject.toml` `[tool.ruff]`): ```bash -# Format code with black -black dns-server/ dns-client/ +# Format code with ruff +ruff format dns-server/app dns-server/tests -# Lint with flake8 -flake8 dns-server/ dns-client/ +# Lint with ruff +ruff check dns-server/app --select=F,E9,B # Type checking with mypy mypy dns-server/bins/server.py - -# Sort imports -isort dns-server/ dns-client/ ``` ## Contributing Process @@ -203,14 +199,14 @@ We follow PEP 8 with some modifications: def process_dns_query(domain: str, record_type: str = "A") -> Dict[str, Any]: """Process a DNS query for the given domain and record type. - + Args: domain: The domain name to query record_type: The DNS record type (default: A) - + Returns: Dictionary containing the DNS response - + Raises: ValueError: If domain is invalid DNSException: If query fails @@ -295,11 +291,11 @@ class TestDNSHandler: """Test that valid token allows DNS query.""" handler = DNSHandler() handler.headers = {"Authorization": "Bearer valid-token"} - + with patch.object(handler, 'check_token_permission_new') as mock_check: mock_check.return_value = True result = handler.do_GET() - + mock_check.assert_called_once() assert result is not None @@ -307,10 +303,10 @@ class TestDNSHandler: """Test that invalid token denies DNS query.""" handler = DNSHandler() handler.headers = {"Authorization": "Bearer invalid-token"} - + with patch.object(handler, 'send_response') as mock_response: handler.do_GET() - + mock_response.assert_called_with(403) @pytest.mark.parametrize("domain,expected", [ @@ -363,26 +359,26 @@ pytest -n auto ```python def authenticate_token(token: str, domain: str) -> bool: """Authenticate token for domain access. - + This function checks if the provided token has permission to access the specified domain. It supports both exact domain matches and wildcard permissions. - + Args: token: The authentication token to validate domain: The domain name being accessed - + Returns: True if token has permission, False otherwise - + Raises: DatabaseError: If database connection fails ValidationError: If inputs are invalid - + Example: >>> authenticate_token("abc123", "example.com") True - >>> authenticate_token("invalid", "example.com") + >>> authenticate_token("invalid", "example.com") False """ pass @@ -397,23 +393,23 @@ Document all API endpoints: @action.uses(db) def api_create_token(): """Create a new authentication token. - + POST /dns_console/api/tokens - + Request Body: { "name": "Token name", - "description": "Token description", + "description": "Token description", "domains": ["example.com", "*.test.com"] } - + Response: { "success": true, "token": "generated-token-value", "id": 123 } - + Error Responses: 400: Invalid request data 409: Token name already exists @@ -498,7 +494,7 @@ Brief description of the problem ## Reproduction Steps 1. Step one -2. Step two +2. Step two 3. Step three ## Expected Behavior @@ -597,7 +593,7 @@ We use Semantic Versioning (semver): ### Release Timeline - **Major releases**: Quarterly -- **Minor releases**: Monthly +- **Minor releases**: Monthly - **Patch releases**: As needed - **Security patches**: Immediately @@ -641,7 +637,7 @@ We recognize contributions in several ways: Regular contributors may be invited to become maintainers: 1. **Active contributor** for 6+ months -2. **High-quality contributions** +2. **High-quality contributions** 3. **Community involvement** 4. **Technical expertise** 5. **Alignment with project values** @@ -669,9 +665,9 @@ Include copyright notice in new files: ```python # Copyright (c) 2024 Penguin Technologies Group LLC -# +# # This file is part of Squawk. -# +# # Squawk is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 7e92456d..4147c7f7 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -108,10 +108,11 @@ pip install -r requirements-dev.txt { "python.defaultInterpreterPath": "./dns-server/venv/bin/python", "python.linting.enabled": true, - "python.linting.flake8Enabled": true, "python.linting.mypyEnabled": true, - "python.formatting.provider": "black", - "python.formatting.blackArgs": ["--line-length", "88"], + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true + }, "python.testing.pytestEnabled": true, "python.testing.pytestArgs": ["tests/"], "files.exclude": { @@ -128,10 +129,10 @@ pip install -r requirements-dev.txt ```python # PyCharm configuration - Interpreter: Project venv Python -- Code style: Black (88 char line length) +- Code style: ruff format (120 char line length, see pyproject.toml) - Test runner: pytest - Type checker: mypy -- Linter: flake8 +- Linter: ruff ``` ### Environment Variables @@ -596,69 +597,42 @@ pytest --pdb ### Linting Configuration -```ini -# setup.cfg -[flake8] -max-line-length = 88 -extend-ignore = E203, W503 -exclude = - .git, - __pycache__, - venv, - .venv, - build, - dist - -[mypy] -python_version = 3.8 -warn_return_any = True -warn_unused_configs = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_redundant_casts = True -warn_unused_ignores = True -``` +ruff supersedes flake8/black/isort (one tool for lint + import-sort + format). +See the actual, current config in `pyproject.toml` at the repo root: ```toml -# pyproject.toml -[tool.black] -line-length = 88 -target-version = ['py38'] -include = '\.pyi?$' -extend-exclude = ''' -/( - # directories - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | build - | dist -)/ -''' - -[tool.isort] -profile = "black" -line_length = 88 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true +# pyproject.toml (excerpt -- see the real file for the full, current config) +[tool.ruff] +target-version = "py313" +line-length = 120 + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "N", "D", "UP", "B", "ASYNC", "S"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.mypy] +python_version = "3.13" +warn_return_any = true +warn_unused_configs = true +warn_unused_ignores = true +check_untyped_defs = false +no_implicit_optional = true +warn_redundant_casts = true +warn_no_return = true ``` ### Pre-commit Hooks +See the actual, current hook set in `.pre-commit-config.yaml` at the repo +root -- this is illustrative only, and will drift if hand-copied: + ```yaml -# .pre-commit-config.yaml +# .pre-commit-config.yaml (excerpt) repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -666,26 +640,19 @@ repos: - id: check-added-large-files - id: check-merge-conflict - - repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - - - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.4 hooks: - - id: flake8 + - id: ruff + args: [--select=F,E9,B, --fix] + - id: ruff-format + stages: [manual] - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.0.1 + - repo: local hooks: - id: mypy - additional_dependencies: [types-all] + name: Type check with mypy (advisory, run via make type-check) + stages: [manual] ``` ### Code Documentation Standards diff --git a/docs/PRE_COMMIT.md b/docs/PRE_COMMIT.md index 2b81e544..d94a613e 100644 --- a/docs/PRE_COMMIT.md +++ b/docs/PRE_COMMIT.md @@ -30,7 +30,7 @@ This script will: Before committing, run in this order (or use `./scripts/pre-commit/pre-commit.sh`): ### Foundation Checks -- [ ] **Linters**: `npm run lint` (React), `black . && flake8 . && mypy .` (Python), `golangci-lint run` (Go) +- [ ] **Linters**: `npm run lint` (React), `ruff check --select=F,E9,B . && mypy .` (Python), `golangci-lint run` (Go) - [ ] **Security scans**: `npm audit`, `gosec ./...`, `bandit -r .` (per language) - [ ] **No secrets**: Verify no credentials, API keys, tokens, or LDAP passwords in code @@ -92,10 +92,9 @@ Before committing, run in this order (or use `./scripts/pre-commit/pre-commit.sh **Linting**: ```bash -black . # Format code -isort . # Sort imports -flake8 . # Check style -mypy . # Type checking +ruff format . # Format code + sort imports +ruff check --select=F,E9,B . # Check style (blocking subset) +mypy . # Type checking ``` **Security**: @@ -366,7 +365,7 @@ Before committing changes to network services: # 1. Make code changes # 2. Test locally npm run lint && npm test # Frontend -pytest --cov && black . && bandit -r . # Python services +pytest --cov && ruff check --select=F,E9,B . && bandit -r . # Python services golangci-lint run && go test -race ./... # Go client # 3. Run smoke tests diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 77d0d412..43a6acca 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -840,4 +840,4 @@ LOG_FILE=/var/log/squawk/webui.log ## License -GNU AGPL v3 - See LICENSE.md for details \ No newline at end of file +GNU AGPL v3 - See LICENSE.md for details diff --git a/docs/STANDARDS.md b/docs/STANDARDS.md index c6a2e9c2..2042e3de 100644 --- a/docs/STANDARDS.md +++ b/docs/STANDARDS.md @@ -91,7 +91,7 @@ Before you commit, run this magic command: Here's what it checks: -- [ ] Linters pass (flake8, eslint, golangci-lint, ansible-lint) +- [ ] Linters pass (ruff, eslint, golangci-lint, ansible-lint) - [ ] Security scans are clean (gosec, bandit, npm audit, Trivy) - [ ] No secrets leaked into code - [ ] Smoke tests pass (build, run, API, UI loads) diff --git a/docs/TOKEN_MANAGEMENT.md b/docs/TOKEN_MANAGEMENT.md index 15581c70..12211367 100644 --- a/docs/TOKEN_MANAGEMENT.md +++ b/docs/TOKEN_MANAGEMENT.md @@ -201,7 +201,7 @@ If you have an existing single-token setup: Token: dev-token Domains: *.dev.example.com, localhost -Token: prod-token +Token: prod-token Domains: *.example.com, *.api.example.com Token: monitoring-token @@ -244,4 +244,4 @@ Domains: db.example.com, db-replica.example.com For issues or questions: - Check the logs at `/dns_console/logs` - Review the README.md for general setup -- Contact support at info@penguintech.group \ No newline at end of file +- Contact support at info@penguintech.group diff --git a/docs/USAGE.md b/docs/USAGE.md index 5dde363c..9a5ac790 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -178,18 +178,18 @@ helm install squawk penguintech/squawk \ # main.tf module "squawk_dns" { source = "github.com/PenguinCloud/terraform-squawk" - + dns_port = 8443 console_port = 8000 enable_ssl = true cert_path = "/certs/server.crt" key_path = "/certs/server.key" - + database = { type = "postgres" url = "postgresql://user:pass@db.example.com/squawk" } - + tokens = [ { name = "production" @@ -211,10 +211,10 @@ module "squawk_dns" { volumes: # Database storage (required) - /data/db:/app/data/db - + # Configuration files (required) - /data/config:/app/config - + # SSL certificates (required for HTTPS) - /data/certs:/app/certs ``` @@ -225,13 +225,13 @@ volumes: volumes: # Custom py4web apps - /custom/apps:/app/web/apps - + # Log files - /var/log/squawk:/app/logs - + # Cache directory - /data/cache:/app/cache - + # Backup directory - /data/backups:/app/backups ``` @@ -417,19 +417,19 @@ services: environment: - NODE_ID=1 - CLUSTER_NODES=dns2,dns3 - + dns2: image: penguintech/squawk:latest environment: - NODE_ID=2 - CLUSTER_NODES=dns1,dns3 - + dns3: image: penguintech/squawk:latest environment: - NODE_ID=3 - CLUSTER_NODES=dns1,dns2 - + haproxy: image: haproxy:latest ports: @@ -450,11 +450,11 @@ class CustomAuthPlugin: if self.check_oauth(token): return True return False - + def check_ldap(self, token): # LDAP authentication pass - + def check_oauth(self, token): # OAuth validation pass diff --git a/docs/WORKFLOWS.md b/docs/WORKFLOWS.md index 0d3b0890..89d382f5 100644 --- a/docs/WORKFLOWS.md +++ b/docs/WORKFLOWS.md @@ -315,14 +315,13 @@ Before pushing code: ```bash # Install dependencies pip install -r requirements.txt -pip install bandit[toml] black isort flake8 mypy pytest +pip install bandit[toml] ruff mypy pytest -# Format -black . -isort . +# Format (ruff supersedes black/isort) +ruff format . # Lint -flake8 . +ruff check --select=F,E9,B . mypy . # Security diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 823c8036..e81a37b4 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -17,7 +17,7 @@ theme: primary: blue accent: blue toggle: - icon: material/brightness-7 + icon: material/brightness-7 name: Switch to dark mode # Palette toggle for dark mode @@ -27,7 +27,7 @@ theme: toggle: icon: material/brightness-4 name: Switch to light mode - + features: - navigation.tabs - navigation.sections @@ -95,4 +95,4 @@ extra: extra_css: - stylesheets/extra.css -copyright: Copyright © 2025 Penguin Technologies. All rights reserved. \ No newline at end of file +copyright: Copyright © 2025 Penguin Technologies. All rights reserved. diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index fe449d12..ade02b97 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -79,12 +79,12 @@ th { .hero-banner { padding: 1.5rem; } - + .hero-banner h2 { font-size: 1.5rem; } - + .hero-banner p { font-size: 1rem; } -} \ No newline at end of file +} diff --git a/entrypoint.yml b/entrypoint.yml index 12bf84b2..4d41f7bb 100644 --- a/entrypoint.yml +++ b/entrypoint.yml @@ -36,4 +36,4 @@ tags: - build loop: - - test \ No newline at end of file + - test diff --git a/install.py b/install.py index 2b8a73f6..e9cbcd99 100644 --- a/install.py +++ b/install.py @@ -22,7 +22,7 @@ def __init__(self): self.install_path = self.get_install_path() self.config_path = self.get_config_path() self.service_name = "squawk-dns" - + def check_admin_privileges(self): """Check if running with admin/root privileges""" if self.system == "Windows": @@ -33,7 +33,7 @@ def check_admin_privileges(self): return False else: return os.geteuid() == 0 - + def get_install_path(self): """Get installation path based on OS""" if self.system == "Windows": @@ -42,14 +42,14 @@ def get_install_path(self): return Path("/usr/local/squawk") else: # Linux return Path("/opt/squawk") - + def get_config_path(self): """Get configuration path based on OS""" if self.system == "Windows": return Path(os.environ.get('PROGRAMDATA', 'C:\\ProgramData')) / "Squawk" else: return Path("/etc/squawk") - + def install_dependencies(self): """Install Python dependencies""" print("Installing Python dependencies...") @@ -61,39 +61,39 @@ def install_dependencies(self): "pystray>=0.19.5", "Pillow>=10.0.0" ] - + for req in requirements: subprocess.run([sys.executable, "-m", "pip", "install", req], check=True) - + def create_directories(self): """Create installation directories""" print(f"Creating directories at {self.install_path}...") self.install_path.mkdir(parents=True, exist_ok=True) self.config_path.mkdir(parents=True, exist_ok=True) - + # Create log directory log_path = self.install_path / "logs" log_path.mkdir(exist_ok=True) - + def copy_files(self): """Copy application files to installation directory""" print("Copying application files...") - + # Copy client files client_src = Path(__file__).parent / "squawk-client" if client_src.exists(): shutil.copytree(client_src / "bins", self.install_path / "bins", dirs_exist_ok=True) shutil.copytree(client_src / "libs", self.install_path / "libs", dirs_exist_ok=True) - + # Make scripts executable on Unix systems if self.system != "Windows": for script in (self.install_path / "bins").glob("*.py"): script.chmod(0o755) - + def create_config(self): """Create default configuration file""" print("Creating configuration file...") - + config = { "dns_server_url": os.getenv("SQUAWK_SERVER_URL", "https://dns.google/resolve"), "auth_token": os.getenv("SQUAWK_AUTH_TOKEN", ""), @@ -108,19 +108,19 @@ def create_config(self): "valkey_url": os.getenv("VALKEY_URL", ""), "log_level": os.getenv("LOG_LEVEL", "INFO") } - + config_file = self.config_path / "config.yaml" with open(config_file, 'w') as f: import yaml yaml.dump(config, f, default_flow_style=False) - + print(f"Configuration saved to {config_file}") return config_file - + def install_windows_service(self): """Install Windows service""" print("Installing Windows service...") - + # Create service wrapper script service_script = self.install_path / "service.py" service_content = f''' @@ -140,32 +140,32 @@ class SquawkDNSService(win32serviceutil.ServiceFramework): _svc_name_ = "SquawkDNS" _svc_display_name_ = "Squawk DNS Client" _svc_description_ = "Local DNS resolver with DoH support" - + def __init__(self, args): win32serviceutil.ServiceFramework.__init__(self, args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) socket.setdefaulttimeout(60) self.running = True - + def SvcStop(self): self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) win32event.SetEvent(self.hWaitStop) self.running = False - + def SvcDoRun(self): servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE, servicemanager.PYS_SERVICE_STARTED, (self._svc_name_, '')) self.main() - + def main(self): config = load_config(r"{self.config_path / 'config.yaml'}") - + dns_client = DNSOverHTTPSClient( config.get('dns_server_url'), config.get('auth_token') ) - + forwarder = DNSForwarder( dns_client, udp_port=config.get('udp_port', 53), @@ -173,13 +173,13 @@ def main(self): listen_udp=config.get('listen_udp', True), listen_tcp=config.get('listen_tcp', False) ) - + # Run in a thread import threading forwarder_thread = threading.Thread(target=forwarder.start) forwarder_thread.daemon = True forwarder_thread.start() - + # Wait for stop signal while self.running: time.sleep(1) @@ -187,25 +187,25 @@ def main(self): if __name__ == '__main__': win32serviceutil.HandleCommandLine(SquawkDNSService) ''' - + with open(service_script, 'w') as f: f.write(service_content) - + # Install pywin32 subprocess.run([sys.executable, "-m", "pip", "install", "pywin32"], check=True) - + # Install the service subprocess.run([sys.executable, str(service_script), "install"], check=True) - + # Configure service to start automatically subprocess.run(["sc", "config", "SquawkDNS", "start=", "auto"], check=True) - + print("Windows service installed successfully") - + def install_macos_daemon(self): """Install macOS launchd daemon""" print("Installing macOS daemon...") - + plist_content = f''' @@ -232,24 +232,24 @@ def install_macos_daemon(self): {self.install_path} ''' - + plist_path = Path("/Library/LaunchDaemons/com.squawk.dns.plist") with open(plist_path, 'w') as f: f.write(plist_content) - + # Set correct permissions os.chown(plist_path, 0, 0) os.chmod(plist_path, 0o644) - + # Load the daemon subprocess.run(["launchctl", "load", str(plist_path)], check=True) - + print("macOS daemon installed successfully") - + def install_systemd_service(self): """Install systemd service for Linux""" print("Installing systemd service...") - + service_content = f'''[Unit] Description=Squawk DNS Client After=network.target @@ -274,21 +274,21 @@ def install_systemd_service(self): [Install] WantedBy=multi-user.target''' - + service_path = Path("/etc/systemd/system/squawk-dns.service") with open(service_path, 'w') as f: f.write(service_content) - + # Reload systemd and enable service subprocess.run(["systemctl", "daemon-reload"], check=True) subprocess.run(["systemctl", "enable", "squawk-dns.service"], check=True) - + print("Systemd service installed successfully") - + def configure_system_dns(self): """Configure system to use local DNS resolver""" print("Configuring system DNS...") - + if self.system == "Windows": # Configure Windows DNS print("Configuring Windows DNS settings...") @@ -297,18 +297,18 @@ def configure_system_dns(self): ["netsh", "interface", "ip", "show", "config"], capture_output=True, text=True ) - + # Set DNS for primary adapter subprocess.run([ "netsh", "interface", "ip", "set", "dns", "name=\"Ethernet\"", "static", "127.0.0.1", "primary" ], check=False) - + subprocess.run([ "netsh", "interface", "ip", "set", "dns", "name=\"Wi-Fi\"", "static", "127.0.0.1", "primary" ], check=False) - + elif self.system == "Darwin": # Configure macOS DNS print("Configuring macOS DNS settings...") @@ -317,19 +317,19 @@ def configure_system_dns(self): ["networksetup", "-listallnetworkservices"], capture_output=True, text=True ) - + services = [s.strip() for s in result.stdout.split('\n')[1:] if s.strip()] - + for service in services: if not service.startswith('*'): subprocess.run([ "networksetup", "-setdnsservers", service, "127.0.0.1" ], check=False) - + else: # Configure Linux DNS print("Configuring Linux DNS settings...") - + # Check if using systemd-resolved if Path("/etc/systemd/resolved.conf").exists(): # Configure systemd-resolved @@ -339,16 +339,16 @@ def configure_system_dns(self): """ with open("/etc/systemd/resolved.conf.d/squawk.conf", 'w') as f: f.write(resolved_conf) - + subprocess.run(["systemctl", "restart", "systemd-resolved"], check=False) - + # Traditional resolv.conf else: # Backup original resolv.conf resolv_conf = Path("/etc/resolv.conf") if resolv_conf.exists(): shutil.copy(resolv_conf, "/etc/resolv.conf.backup") - + # Write new resolv.conf with open(resolv_conf, 'w') as f: f.write("# Managed by Squawk DNS\n") @@ -356,23 +356,23 @@ def configure_system_dns(self): f.write("# Fallback DNS servers\n") f.write("nameserver 8.8.8.8\n") f.write("nameserver 8.8.4.4\n") - + def start_service(self): """Start the installed service""" print("Starting service...") - + if self.system == "Windows": subprocess.run(["net", "start", "SquawkDNS"], check=False) elif self.system == "Darwin": subprocess.run(["launchctl", "start", "com.squawk.dns"], check=False) else: subprocess.run(["systemctl", "start", "squawk-dns.service"], check=False) - + def install(self): """Main installation process""" print(f"Squawk DNS Client Installer - {self.system}") print("=" * 50) - + if not self.is_admin: print("ERROR: This installer requires administrator/root privileges") if self.system == "Windows": @@ -380,20 +380,20 @@ def install(self): else: print("Please run with sudo") sys.exit(1) - + try: # Step 1: Install dependencies self.install_dependencies() - + # Step 2: Create directories self.create_directories() - + # Step 3: Copy files self.copy_files() - + # Step 4: Create configuration self.create_config() - + # Step 5: Install service/daemon if self.system == "Windows": self.install_windows_service() @@ -401,20 +401,20 @@ def install(self): self.install_macos_daemon() else: self.install_systemd_service() - + # Step 6: Configure system DNS self.configure_system_dns() - + # Step 7: Start service self.start_service() - + print("\n" + "=" * 50) print("Installation completed successfully!") print(f"Configuration file: {self.config_path / 'config.yaml'}") print(f"Logs directory: {self.install_path / 'logs'}") print("\nThe DNS service is now running and your system is configured to use it.") print("\nTo configure the service, edit the configuration file and restart the service.") - + if self.system == "Windows": print("\nService management:") print(" Start: net start SquawkDNS") @@ -431,19 +431,19 @@ def install(self): print(" Stop: sudo systemctl stop squawk-dns") print(" Status: sudo systemctl status squawk-dns") print(" Logs: sudo journalctl -u squawk-dns -f") - + except Exception as e: print(f"\nERROR: Installation failed: {e}") sys.exit(1) - + def uninstall(self): """Uninstall the service""" print("Uninstalling Squawk DNS Client...") - + if not self.is_admin: print("ERROR: Uninstallation requires administrator/root privileges") sys.exit(1) - + try: # Stop and remove service if self.system == "Windows": @@ -458,46 +458,46 @@ def uninstall(self): subprocess.run(["systemctl", "disable", "squawk-dns.service"], check=False) Path("/etc/systemd/system/squawk-dns.service").unlink(missing_ok=True) subprocess.run(["systemctl", "daemon-reload"], check=False) - + # Restore DNS settings if self.system == "Linux" and Path("/etc/resolv.conf.backup").exists(): shutil.move("/etc/resolv.conf.backup", "/etc/resolv.conf") - + # Remove directories if self.install_path.exists(): shutil.rmtree(self.install_path) if self.config_path.exists(): shutil.rmtree(self.config_path) - + print("Uninstallation completed successfully!") - + except Exception as e: print(f"ERROR: Uninstallation failed: {e}") sys.exit(1) def main(): parser = argparse.ArgumentParser(description="Squawk DNS Client Installer") - parser.add_argument('action', choices=['install', 'uninstall'], + parser.add_argument('action', choices=['install', 'uninstall'], help='Action to perform') - parser.add_argument('--server-url', help='DNS server URL', + parser.add_argument('--server-url', help='DNS server URL', default=os.getenv('SQUAWK_SERVER_URL')) parser.add_argument('--auth-token', help='Authentication token', default=os.getenv('SQUAWK_AUTH_TOKEN')) - + args = parser.parse_args() - + # Set environment variables if provided if args.server_url: os.environ['SQUAWK_SERVER_URL'] = args.server_url if args.auth_token: os.environ['SQUAWK_AUTH_TOKEN'] = args.auth_token - + installer = SquawkInstaller() - + if args.action == 'install': installer.install() else: installer.uninstall() if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/manager/backend/app/blueprints/analytics.py b/manager/backend/app/blueprints/analytics.py index 15897fd9..ee08841b 100644 --- a/manager/backend/app/blueprints/analytics.py +++ b/manager/backend/app/blueprints/analytics.py @@ -43,7 +43,6 @@ def get_query_analytics(): # Parse query parameters period_hours = int(request.args.get('period', 24)) server_id = request.args.get('server_id') - team_id = request.args.get('team_id') since = datetime.utcnow() - timedelta(hours=period_hours) diff --git a/manager/backend/app/blueprints/dhcp.py b/manager/backend/app/blueprints/dhcp.py index 373a185b..31de3211 100644 --- a/manager/backend/app/blueprints/dhcp.py +++ b/manager/backend/app/blueprints/dhcp.py @@ -362,7 +362,6 @@ def list_reservations(): - pool_id: Filter by pool """ db = current_app.db - user = get_current_user() query = db.dhcp_reservation.id > 0 diff --git a/manager/backend/app/blueprints/machine_clients.py b/manager/backend/app/blueprints/machine_clients.py index 2c7f67b8..5934c530 100644 --- a/manager/backend/app/blueprints/machine_clients.py +++ b/manager/backend/app/blueprints/machine_clients.py @@ -4,10 +4,9 @@ """ from flask import Blueprint, request, jsonify, current_app -from app.middleware.auth import token_required, get_current_user +from app.middleware.auth import token_required from app.middleware.rbac import requires_system_admin from app.services.auth_service import AuthService -from app.services.scopes import SUPERADMIN_SCOPE from app.utils.decorators import validate_json, audit_log from app.utils.domain_validation import validate_allowed_domains from datetime import datetime @@ -129,7 +128,7 @@ def create_machine_client(): return jsonify({'error': error_msg}), 400 # Validate scopes exist in the system - from app.services.scopes import ROLE_SCOPES, _READ_SCOPES + from app.services.scopes import ROLE_SCOPES all_valid_scopes = set() for scope_bundle in ROLE_SCOPES.values(): all_valid_scopes.update(scope_bundle) diff --git a/manager/backend/app/blueprints/mfa.py b/manager/backend/app/blueprints/mfa.py index ef97aaba..24e0a026 100644 --- a/manager/backend/app/blueprints/mfa.py +++ b/manager/backend/app/blueprints/mfa.py @@ -10,7 +10,6 @@ from app.services.auth_service import AuthService from app.services.mfa_service import MFAService from app.utils.decorators import validate_json, audit_log -from werkzeug.exceptions import BadRequest mfa_bp = Blueprint('mfa', __name__) diff --git a/manager/backend/app/blueprints/oidc_trust_anchors.py b/manager/backend/app/blueprints/oidc_trust_anchors.py index e9270b95..93c9861b 100644 --- a/manager/backend/app/blueprints/oidc_trust_anchors.py +++ b/manager/backend/app/blueprints/oidc_trust_anchors.py @@ -4,7 +4,7 @@ """ from flask import Blueprint, request, jsonify, current_app -from app.middleware.auth import token_required, get_current_user +from app.middleware.auth import token_required from app.middleware.rbac import requires_system_admin from app.utils.decorators import validate_json, audit_log from app.utils.domain_validation import validate_allowed_domains diff --git a/manager/backend/app/blueprints/saml.py b/manager/backend/app/blueprints/saml.py index 19baa17e..8860cf9b 100644 --- a/manager/backend/app/blueprints/saml.py +++ b/manager/backend/app/blueprints/saml.py @@ -13,7 +13,6 @@ from flask import Blueprint, request, jsonify, current_app, make_response, redirect from app.services.saml_service import SAMLService, SAMLConfig from app.services.auth_service import AuthService -from app.utils.decorators import validate_json saml_bp = Blueprint('saml', __name__) diff --git a/manager/backend/app/blueprints/scim.py b/manager/backend/app/blueprints/scim.py index 80d9d3da..593d53fc 100644 --- a/manager/backend/app/blueprints/scim.py +++ b/manager/backend/app/blueprints/scim.py @@ -11,7 +11,7 @@ from dataclasses import asdict, dataclass, field from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from flask import Blueprint, request, jsonify, current_app import logging @@ -231,9 +231,12 @@ def list_users(): return result db = current_app.db - from flask import g + # TODO(security): `g.tenant` is resolved but not applied as a filter on + # the auth_user queries below (list-all and username-filter branches) -- + # this endpoint currently returns users across all tenants. Flagged + # during git-hooks lint cleanup; needs a dedicated security-tagged fix, + # not a drive-by change here. See security.md Tenant Isolation. - tenant = g.tenant start_index = request.args.get('startIndex', 1, type=int) count = request.args.get('count', 100, type=int) filter_param = request.args.get('filter', '') @@ -360,7 +363,7 @@ def create_user(): user = db(db.auth_user.username == username).select().first() scim_user = _user_record_to_scim(user) - logger.info(f"SCIM user created", extra={"user_id": user.id, "username": username}) + logger.info("SCIM user created", extra={"user_id": user.id, "username": username}) return jsonify(scim_user.to_dict()), 201 @@ -405,7 +408,7 @@ def update_user(user_id: str): # Reload user from database to get updated values user = db.auth_user[int(user_id)] scim_user = _user_record_to_scim(user) - logger.info(f"SCIM user updated", extra={"user_id": user.id}) + logger.info("SCIM user updated", extra={"user_id": user.id}) return jsonify(scim_user.to_dict()), 200 @@ -464,7 +467,7 @@ def patch_user(user_id: str): # Reload user from database to get updated values user = db.auth_user[int(user_id)] scim_user = _user_record_to_scim(user) - logger.info(f"SCIM user patched", extra={"user_id": user.id}) + logger.info("SCIM user patched", extra={"user_id": user.id}) return jsonify(scim_user.to_dict()), 200 @@ -490,7 +493,7 @@ def delete_user(user_id: str): db(db.auth_user.id == user.id).update(active=False) db.commit() - logger.info(f"SCIM user deprovisioned", extra={"user_id": user.id, "username": user.username}) + logger.info("SCIM user deprovisioned", extra={"user_id": user.id, "username": user.username}) return '', 204 @@ -516,7 +519,6 @@ def mint_scim_token(): } """ # Check user has admin:super scope - from flask import g from app.middleware import verify_jwt, has_scope # Verify caller is authenticated user (not SCIM bearer) @@ -536,7 +538,7 @@ def mint_scim_token(): plaintext, token_hash = SCIMTokenService.create_token(description, tenant) token_id = SCIMTokenService.store_token(plaintext, description, tenant) - logger.info(f"SCIM token minted", extra={"token_id": token_id, "tenant": tenant}) + logger.info("SCIM token minted", extra={"token_id": token_id, "tenant": tenant}) return { 'id': token_id, @@ -549,7 +551,6 @@ def mint_scim_token(): @scim_bp.route('/admin/tokens/', methods=['DELETE']) def revoke_scim_token(token_id: str): """Revoke a SCIM token (admin only).""" - from flask import g from app.middleware import verify_jwt, has_scope token_payload = verify_jwt(request.headers.get('Authorization', '')) @@ -560,7 +561,7 @@ def revoke_scim_token(token_id: str): if not success: return scim_error('resourceNotFound', f'Token {token_id} not found', 404) - logger.info(f"SCIM token revoked", extra={"token_id": token_id}) + logger.info("SCIM token revoked", extra={"token_id": token_id}) return '', 204 diff --git a/manager/backend/app/models/dhcp.py b/manager/backend/app/models/dhcp.py index 43769709..d11f024e 100644 --- a/manager/backend/app/models/dhcp.py +++ b/manager/backend/app/models/dhcp.py @@ -58,7 +58,7 @@ def define_dhcp_tables(db): db.define_table('dhcp_server', Field('name', 'string', notnull=True, length=100), Field('hostname', 'string', length=255), - Field('listen_address', 'string', length=50, default='0.0.0.0'), + Field('listen_address', 'string', length=50, default='0.0.0.0'), # nosec B104 -- config default for a server field, not a live bind Field('status', 'string', notnull=True, default='offline', requires=lambda value: value in ['online', 'offline', 'degraded']), Field('last_heartbeat', 'datetime'), diff --git a/manager/backend/app/observability.py b/manager/backend/app/observability.py index 51dd8acc..c006af22 100644 --- a/manager/backend/app/observability.py +++ b/manager/backend/app/observability.py @@ -7,7 +7,6 @@ import os import logging -from typing import Optional logger = logging.getLogger(__name__) diff --git a/manager/backend/app/schema.py b/manager/backend/app/schema.py index 46114f06..a8cd85bc 100644 --- a/manager/backend/app/schema.py +++ b/manager/backend/app/schema.py @@ -384,7 +384,7 @@ Column("id", Integer, primary_key=True, autoincrement=True), Column("name", String(100), nullable=False), Column("hostname", String(255)), - Column("listen_address", String(50), server_default="0.0.0.0"), + Column("listen_address", String(50), server_default="0.0.0.0"), # nosec B104 -- config default for a server field, not a live bind Column("status", String(20), nullable=False, server_default="offline"), Column("last_heartbeat", DateTime), Column("version", String(50)), diff --git a/manager/backend/app/services/config_service.py b/manager/backend/app/services/config_service.py index 87704ae4..d2dc1958 100644 --- a/manager/backend/app/services/config_service.py +++ b/manager/backend/app/services/config_service.py @@ -22,8 +22,6 @@ def get_server_config(server_id: int) -> Dict: Returns: Configuration dict with zones, IOC feeds, and settings """ - db = current_app.db - # Get all zones with records zones = ConfigService.get_all_zones() diff --git a/manager/backend/app/services/dpop_service.py b/manager/backend/app/services/dpop_service.py index 09e6fa0d..86a40d63 100644 --- a/manager/backend/app/services/dpop_service.py +++ b/manager/backend/app/services/dpop_service.py @@ -12,12 +12,12 @@ import jwt import hashlib import base64 +import traceback from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Optional, Dict, Tuple, Any +from typing import Optional, Dict, Any from flask import current_app -from cryptography.hazmat.primitives import serialization, hashes -from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding +from cryptography.hazmat.primitives.asymmetric import rsa, ec from cryptography.hazmat.backends import default_backend diff --git a/manager/backend/app/services/ioc_ingestion_service.py b/manager/backend/app/services/ioc_ingestion_service.py index d7e0f067..cde00e4f 100644 --- a/manager/backend/app/services/ioc_ingestion_service.py +++ b/manager/backend/app/services/ioc_ingestion_service.py @@ -266,7 +266,6 @@ async def _parse_text_feed( indicators: List[IOCIndicator] = [] config = config or {} comment_chars = config.get("comment_chars", ["#", ";"]) - skip_localhost = config.get("skip_localhost", False) for line in content.split("\n"): line = line.strip() diff --git a/manager/backend/app/services/saml_service.py b/manager/backend/app/services/saml_service.py index 6b0ae2db..027a2976 100644 --- a/manager/backend/app/services/saml_service.py +++ b/manager/backend/app/services/saml_service.py @@ -24,7 +24,7 @@ import base64 import uuid from typing import Optional -from datetime import datetime, timedelta +from datetime import datetime from dataclasses import dataclass from urllib.parse import urlencode @@ -35,10 +35,9 @@ from flask import current_app # SAML 2.0 XML parsing with defused XML (prevents XXE) -from defusedxml import ElementTree as ET from saml2.response import AuthnResponse from saml2.config import Config as Saml2Config -from saml2 import BINDING_HTTP_REDIRECT, BINDING_HTTP_POST +from saml2 import BINDING_HTTP_POST @dataclass(slots=True) diff --git a/manager/backend/app/services/signing_provider.py b/manager/backend/app/services/signing_provider.py index feb0e8b0..831cccd8 100644 --- a/manager/backend/app/services/signing_provider.py +++ b/manager/backend/app/services/signing_provider.py @@ -199,7 +199,7 @@ def _initialize(self) -> None: raise ImportError( "boto3 is required for AwsKmsProvider. " "Install it with: pip install boto3" - ) + ) from None self._kms_client = boto3.client("kms", region_name=None) self._fetch_public_key() diff --git a/manager/backend/app/services/sso_service.py b/manager/backend/app/services/sso_service.py index 0e9f34cb..89a28973 100644 --- a/manager/backend/app/services/sso_service.py +++ b/manager/backend/app/services/sso_service.py @@ -18,7 +18,7 @@ import hashlib import base64 from typing import Optional, Tuple -from datetime import datetime, timedelta +from datetime import datetime from dataclasses import dataclass from urllib.parse import urlencode @@ -177,11 +177,11 @@ def get_login_attempt(state: str, db): attempt = db(db.sso_login_attempts.opaque_state == state).select().first() if not attempt: - current_app.logger.warning(f"Login attempt not found for state") + current_app.logger.warning("Login attempt not found for state") return None if attempt['used']: - current_app.logger.warning(f"Login attempt already used") + current_app.logger.warning("Login attempt already used") return None age = (datetime.utcnow() - attempt['created_at']).total_seconds() diff --git a/manager/backend/app/services/whois_service.py b/manager/backend/app/services/whois_service.py index 20fb9fe4..8cc9225e 100644 --- a/manager/backend/app/services/whois_service.py +++ b/manager/backend/app/services/whois_service.py @@ -360,7 +360,6 @@ async def _perform_ip_whois(self, ip: str) -> dict[str, Any]: } # Always try RDAP first - rdap_failed = False try: rdap_data = await loop.run_in_executor(None, obj.lookup_rdap) @@ -374,7 +373,6 @@ async def _perform_ip_whois(self, ip: str) -> dict[str, Any]: result["country"] = network["country"] except Exception as rdap_error: logger.debug(f"RDAP lookup failed for {ip}: {rdap_error}") - rdap_failed = True # Always try legacy lookup (test expects both to be called) try: diff --git a/manager/backend/app/utils/crypto.py b/manager/backend/app/utils/crypto.py index f4361717..85f0398d 100644 --- a/manager/backend/app/utils/crypto.py +++ b/manager/backend/app/utils/crypto.py @@ -3,7 +3,7 @@ from __future__ import annotations import hashlib -from typing import Optional, Tuple +from typing import Tuple def compute_kid_from_public_pem(public_pem: str) -> str: diff --git a/manager/backend/app/utils/decorators.py b/manager/backend/app/utils/decorators.py index 0c17ac43..df131421 100644 --- a/manager/backend/app/utils/decorators.py +++ b/manager/backend/app/utils/decorators.py @@ -253,7 +253,7 @@ def decorated_function(*args, **kwargs): # Extract request_id if present in headers or response request_id = request.headers.get('X-Request-ID') - except Exception as e: + except Exception: # Audit the failure, log the error, re-raise outcome = 'failure' status_code = 500 diff --git a/manager/backend/requirements-dev.txt b/manager/backend/requirements-dev.txt index 3a310c32..7e19565e 100644 --- a/manager/backend/requirements-dev.txt +++ b/manager/backend/requirements-dev.txt @@ -1,6 +1,3 @@ # Development/lint tooling for manager backend (not installed in the runtime image) -flake8>=6.1.0 -flake8-bugbear>=24.2.6 -black>=23.9.1 -isort>=5.12.0 +ruff>=0.8.4 mypy>=1.6.1 diff --git a/manager/tests/test_manager_api.py b/manager/tests/test_manager_api.py index d7c9f859..c51b9d4b 100644 --- a/manager/tests/test_manager_api.py +++ b/manager/tests/test_manager_api.py @@ -27,17 +27,17 @@ def client(mock_app): class TestManagerHealthEndpoint: """Test health check endpoint""" - + def test_health_check(self, client): """Test health endpoint returns 200""" with patch('app.db') as mock_db: mock_db.session.execute.return_value = True - + # Mock the health endpoint @client.application.route('/health') def health(): return {'status': 'healthy', 'service': 'manager-backend'}, 200 - + response = client.get('/health') assert response.status_code == 200 data = json.loads(response.data) @@ -46,7 +46,7 @@ def health(): class TestManagerAuthAPI: """Test authentication endpoints""" - + def test_login_success(self, client): """Test successful login""" with patch('app.db') as mock_db: @@ -55,43 +55,43 @@ def test_login_success(self, client): mock_user.id = 1 mock_user.email = 'test@example.com' mock_user.check_password.return_value = True - + mock_db.session.query.return_value.filter_by.return_value.first.return_value = mock_user - + # Mock login endpoint @client.application.route('/api/auth/login', methods=['POST']) def login(): return {'token': 'test-jwt-token', 'user_id': 1}, 200 - + response = client.post('/api/auth/login', json={ 'email': 'test@example.com', 'password': 'password123' }) - + assert response.status_code == 200 data = json.loads(response.data) assert 'token' in data - + def test_login_invalid_credentials(self, client): """Test login with invalid credentials""" with patch('app.db') as mock_db: mock_db.session.query.return_value.filter_by.return_value.first.return_value = None - + @client.application.route('/api/auth/login', methods=['POST']) def login(): return {'error': 'Invalid credentials'}, 401 - + response = client.post('/api/auth/login', json={ 'email': 'wrong@example.com', 'password': 'wrongpassword' }) - + assert response.status_code == 401 class TestManagerDNSServersAPI: """Test DNS servers management endpoints""" - + def test_list_dns_servers(self, client): """Test listing DNS servers""" with patch('app.db') as mock_db: @@ -100,31 +100,31 @@ def test_list_dns_servers(self, client): Mock(id=2, hostname='server2.example.com', status='active') ] mock_db.session.query.return_value.all.return_value = mock_servers - + @client.application.route('/api/dns-servers') def list_servers(): return {'servers': [ {'id': 1, 'hostname': 'server1.example.com', 'status': 'active'}, {'id': 2, 'hostname': 'server2.example.com', 'status': 'active'} ]}, 200 - + response = client.get('/api/dns-servers') assert response.status_code == 200 data = json.loads(response.data) assert len(data['servers']) == 2 - + def test_register_dns_server(self, client): """Test registering new DNS server""" with patch('app.db') as mock_db: @client.application.route('/api/dns-servers/register', methods=['POST']) def register_server(): return {'id': 1, 'status': 'registered'}, 201 - + response = client.post('/api/dns-servers/register', json={ 'hostname': 'newserver.example.com', 'join_key': 'secret-key-123' }) - + assert response.status_code == 201 data = json.loads(response.data) assert data['status'] == 'registered' @@ -132,7 +132,7 @@ def register_server(): class TestManagerConfigAPI: """Test configuration management endpoints""" - + def test_get_global_config(self, client): """Test getting global configuration""" with patch('app.db') as mock_db: @@ -143,29 +143,29 @@ def get_config(): 'enable_ioc_blocking': True, 'log_level': 'INFO' }, 200 - + response = client.get('/api/config/global') assert response.status_code == 200 data = json.loads(response.data) assert 'cache_ttl' in data - + def test_update_global_config(self, client): """Test updating global configuration""" with patch('app.db') as mock_db: @client.application.route('/api/config/global', methods=['PUT']) def update_config(): return {'status': 'updated'}, 200 - + response = client.put('/api/config/global', json={ 'cache_ttl': 600 }) - + assert response.status_code == 200 class TestManagerStatsAPI: """Test statistics endpoints""" - + def test_get_system_stats(self, client): """Test getting system statistics""" with patch('app.db') as mock_db: @@ -176,7 +176,7 @@ def get_stats(): 'active_servers': 5, 'cache_hit_rate': 85.5 }, 200 - + response = client.get('/api/stats/system') assert response.status_code == 200 data = json.loads(response.data) @@ -185,7 +185,7 @@ def get_stats(): class TestManagerIOCAPI: """Test IOC management endpoints""" - + def test_list_ioc_feeds(self, client): """Test listing IOC feeds""" with patch('app.db') as mock_db: @@ -194,19 +194,19 @@ def list_feeds(): return {'feeds': [ {'id': 1, 'name': 'Test Feed', 'url': 'https://example.com/feed.txt'} ]}, 200 - + response = client.get('/api/ioc/feeds') assert response.status_code == 200 data = json.loads(response.data) assert 'feeds' in data - + def test_sync_ioc_feed(self, client): """Test syncing IOC feed""" with patch('app.db') as mock_db: @client.application.route('/api/ioc/feeds//sync', methods=['POST']) def sync_feed(feed_id): return {'status': 'synced', 'entries_added': 150}, 200 - + response = client.post('/api/ioc/feeds/1/sync') assert response.status_code == 200 data = json.loads(response.data) diff --git a/ntp-server/bins/server.py b/ntp-server/bins/server.py index e95d9ccf..95526e42 100644 --- a/ntp-server/bins/server.py +++ b/ntp-server/bins/server.py @@ -659,7 +659,7 @@ async def run(self) -> None: # Create server socket server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("0.0.0.0", self.port)) + server_socket.bind(("0.0.0.0", self.port)) # nosec B104 -- NTS-KE server must accept connections on all container interfaces server_socket.listen(5) server_socket.setblocking(False) @@ -884,7 +884,7 @@ async def run(self) -> None: loop = asyncio.get_event_loop() sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("0.0.0.0", self.port)) + sock.bind(("0.0.0.0", self.port)) # nosec B104 -- UDP NTP server must accept requests on all container interfaces sock.setblocking(False) logger.info(f"UDP NTP server listening on port {self.port}") diff --git a/ntp-server/requirements-dev.txt b/ntp-server/requirements-dev.txt index 690d7761..b403ae26 100644 --- a/ntp-server/requirements-dev.txt +++ b/ntp-server/requirements-dev.txt @@ -1,6 +1,3 @@ # Development/lint tooling for NTP/NTS server (not installed in the runtime image) -flake8>=6.1.0 -flake8-bugbear>=24.2.6 -black>=23.9.1 -isort>=5.12.0 +ruff>=0.8.4 mypy>=1.6.1 diff --git a/pyproject.toml b/pyproject.toml index 88ff72a5..8b22cc8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,44 +1,32 @@ -[tool.black] +[tool.ruff] +# ruff supersedes flake8/black/isort (backend-python.md) -- one tool for +# lint + import-sort + format. line-length kept at 120 (not the canonical +# 100) to match this codebase's existing black/isort history and avoid a +# repo-wide reformat as a side effect of adopting the tool. +target-version = "py313" line-length = 120 -target-version = ['py312'] -include = '\.pyi?$' -extend-exclude = ''' -/( - # directories - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | venv - | _build - | buck-out - | build - | dist - | htmlcov - | migrations - | alembic -)/ -''' - -[tool.isort] -profile = "black" -line_length = 120 -skip_glob = [ - "*/venv/*", - "*/.venv/*", - "*/node_modules/*", - "*/.pytest_cache/*", - "*/htmlcov/*", - "*/.mypy_cache/*", - "*/alembic/*", - "*/migrations/*" +extend-exclude = [ + "venv", ".venv", "*/venv", "*/.venv", "node_modules", + ".pytest_cache", "htmlcov", ".mypy_cache", "migrations", "alembic", + "*/migrations", "*/alembic", ] -known_first_party = ["dns_server", "dns_client", "manager"] -include_trailing_comma = true -use_parentheses = true -ensure_newline_before_comments = true + +[tool.ruff.lint] +# Canonical full rule set (see backend-python.md). Only F/E9/B are wired as +# blocking in .pre-commit-config.yaml today -- the same bar flake8 already +# enforced (E9,F63,F7,F821,F401,B), now via ruff. D/N/UP/ASYNC/S/I/E-full/ +# W-full surface ~1,700 pre-existing findings (mostly missing docstrings and +# pyupgrade modernization) across five services that predate this change and +# are not fixable as a "add git hooks" chore -- run via `ruff check .` +# (advisory, `make lint-full`) and close incrementally. Mirrors the mypy +# manual-stage exception below. +select = ["E", "W", "F", "I", "N", "D", "UP", "B", "ASYNC", "S"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +known-first-party = ["dns_server", "dns_client", "manager"] [tool.mypy] python_version = "3.13" diff --git a/scripts/deploy-alpha.sh b/scripts/deploy-alpha.sh index d7f4c7cc..08d271cd 100755 --- a/scripts/deploy-alpha.sh +++ b/scripts/deploy-alpha.sh @@ -28,8 +28,10 @@ set -euo pipefail # Configuration # ============================================================================= -readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" +SCRIPT_DIR_VALUE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR="$SCRIPT_DIR_VALUE" +PROJECT_ROOT_VALUE="$(dirname "${SCRIPT_DIR}")" +readonly PROJECT_ROOT="$PROJECT_ROOT_VALUE" readonly APP_NAME="${APP_NAME:-squawk}" readonly KUBE_CONTEXT="${KUBE_CONTEXT:-local-alpha}" diff --git a/scripts/deploy-beta.sh b/scripts/deploy-beta.sh index 4d9cdf0d..0978779d 100755 --- a/scripts/deploy-beta.sh +++ b/scripts/deploy-beta.sh @@ -22,7 +22,8 @@ readonly CHART_PATH="./k8s/helm/squawk" readonly IMAGE_REGISTRY="registry-dal2.penguintech.io" readonly KUBE_CONTEXT="dal2-beta" readonly APP_HOST="squawk.penguintech.cloud" -readonly DEFAULT_TAG="beta-$(date +%s)" +DEFAULT_TAG_VALUE="beta-$(date +%s)" +readonly DEFAULT_TAG="$DEFAULT_TAG_VALUE" # Service definitions (docker build contexts) declare -A SERVICES=( @@ -168,7 +169,7 @@ build_and_push() { if [[ -n "$SERVICE" ]]; then # Build specific service if [[ -z "${SERVICES[$SERVICE]:-}" ]]; then - die "Unknown service: $SERVICE. Available: ${!SERVICES[@]}" + die "Unknown service: $SERVICE. Available: ${!SERVICES[*]}" fi build_and_push_image "$SERVICE" else @@ -267,7 +268,8 @@ verify_deployment() { print_step "Health Check Summary:" local unhealthy=0 for pod in $(kubectl get pods -n "$NAMESPACE" -o jsonpath='{.items[*].metadata.name}'); do - local status=$(kubectl get pod "$pod" -n "$NAMESPACE" -o jsonpath='{.status.phase}') + local status + status=$(kubectl get pod "$pod" -n "$NAMESPACE" -o jsonpath='{.status.phase}') if [[ "$status" != "Running" ]]; then print_warning "Pod $pod is $status" ((unhealthy++)) diff --git a/scripts/hooks/check-dockerfile-rootless.sh b/scripts/hooks/check-dockerfile-rootless.sh new file mode 100755 index 00000000..6fa91d03 --- /dev/null +++ b/scripts/hooks/check-dockerfile-rootless.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# check-dockerfile-rootless.sh — fail any Dockerfile that ends up running as root. +# +# devops-containers.md requires a non-root process in every container. An +# explicit, approved exception is allowed but must be annotated so the decision +# is visible in review rather than implied by silence. +# +# Usage: check-dockerfile-rootless.sh ... (invoked by pre-commit) +set -uo pipefail + +status=0 + +for file in "$@"; do + [[ -f "$file" ]] || continue + + # An approved exception suppresses the check for the whole file. + # + # The pattern is deliberately strict. A loose `#.*ROOT EXCEPTION` match + # would also fire on comment-shaped lines that are not Dockerfile comments + # at all — most importantly heredoc bodies, where the token can be smuggled + # into file content and silently disable the check: + # + # RUN cat <<'EOF' > /etc/motd + # # ROOT EXCEPTION (approved) + # EOF + # + # Requiring the trailing colon plus a non-empty reason means an exception + # has to be written deliberately, and heredoc payloads do not match by + # accident. A bypass is also never silent — see the notice below. + exception="$(grep -nE '^[[:space:]]*#[[:space:]]*ROOT EXCEPTION \(approved\):[[:space:]]*[^[:space:]]' "$file" | head -1)" + if [[ -n "$exception" ]]; then + echo "$file: rootless check BYPASSED by approved exception" + echo " ${exception}" + continue + fi + + # A malformed annotation must not fail open — it reads as an exception to a + # human but matches nothing above, so call it out explicitly. + if grep -qE '^[[:space:]]*#.*ROOT EXCEPTION' "$file"; then + echo "$file: malformed ROOT EXCEPTION annotation — not honoured" + echo " Required form: # ROOT EXCEPTION (approved): " + status=1 + continue + fi + + # The effective user is whatever the last USER instruction sets. Strip any + # group suffix ("appuser:appgroup") before deciding. + last_user="$(grep -iE '^[[:space:]]*USER[[:space:]]+' "$file" | tail -1 | awk '{print $2}')" + last_user="${last_user%%:*}" + + if [[ -z "$last_user" ]]; then + echo "$file: no USER instruction — container would run as root" + echo " Add a non-root USER, or annotate: # ROOT EXCEPTION (approved): " + status=1 + elif [[ "$last_user" == \$* || "$last_user" == *'${'* ]]; then + # Resolved at build time from an ARG/ENV — cannot be verified statically. + echo "$file: final USER is build-arg '$last_user' — cannot verify it is non-root" + echo " Use a literal non-root USER, or annotate: # ROOT EXCEPTION (approved): " + status=1 + elif [[ "$last_user" == "root" || "$last_user" == "0" || "$last_user" == 0:* ]]; then + echo "$file: final USER is '$last_user' — container runs as root" + echo " Switch to a non-root user, or annotate: # ROOT EXCEPTION (approved): " + status=1 + fi +done + +exit "$status" diff --git a/scripts/init-postgres.sql b/scripts/init-postgres.sql index 4306c9c4..cff8edf8 100644 --- a/scripts/init-postgres.sql +++ b/scripts/init-postgres.sql @@ -60,7 +60,7 @@ CREATE INDEX IF NOT EXISTS idx_query_logs_status ON query_logs(status); -- Create a view for easier token permission queries CREATE OR REPLACE VIEW token_permissions AS -SELECT +SELECT t.id as token_id, t.token, t.name as token_name, @@ -74,7 +74,7 @@ JOIN domains d ON td.domain_id = d.id; -- Create a view for query statistics CREATE OR REPLACE VIEW query_statistics AS -SELECT +SELECT t.name as token_name, ql.status, COUNT(*) as query_count, @@ -90,4 +90,4 @@ GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO squawk_user; -- Print success message \echo 'PostgreSQL database initialization completed successfully!' -\echo 'Schema created. Use scripts/dev-seed.sql for test data.' \ No newline at end of file +\echo 'Schema created. Use scripts/dev-seed.sql for test data.' diff --git a/scripts/install-pre-commit.sh b/scripts/install-pre-commit.sh new file mode 100755 index 00000000..4fa6becb --- /dev/null +++ b/scripts/install-pre-commit.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# ============================================================================= +# install-pre-commit.sh — Install the pre-commit framework and wire up hooks +# +# Installs pre-commit system-wide (macOS, Ubuntu/Debian, WSL, Fedora/RHEL) and +# registers both hook types in the current repo: +# pre-commit — fast lint + secrets checks +# pre-push — heavier security scans +# +# Usage: +# ./install-pre-commit.sh # Install framework + hooks +# ./install-pre-commit.sh --hooks-only # Skip the framework install +# ./install-pre-commit.sh --verify # Report state, change nothing +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck source=lib/detect-os.sh +source "$SCRIPT_DIR/lib/detect-os.sh" +detect_os + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[0;33m'; CYAN='\033[0;36m'; NC='\033[0m' +info() { echo -e "${CYAN}[INFO]${NC} $*"; } +ok() { echo -e "${GREEN}[OK]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +err() { echo -e "${RED}[ERR]${NC} $*" >&2; exit 1; } + +install_framework() { + if command -v pre-commit >/dev/null 2>&1; then + ok "pre-commit already installed ($(pre-commit --version))" + return + fi + + info "Installing pre-commit via $PKG_MANAGER..." + case "$PKG_MANAGER" in + brew) + brew install pre-commit + ;; + apt) + # Same path for native Ubuntu/Debian and WSL — no WSL special-casing needed. + sudo apt-get update -q + sudo apt-get install -y -q pre-commit + ;; + dnf|yum) + sudo "$PKG_MANAGER" install -y pre-commit + ;; + *) + warn "Unknown package manager '$PKG_MANAGER' — falling back to uv" + ;; + esac + + # Distro packages lag; fall back to a userspace install rather than failing. + if ! command -v pre-commit >/dev/null 2>&1; then + if command -v uv >/dev/null 2>&1; then + info "Package manager did not provide pre-commit — installing via uv" + uv tool install pre-commit + else + python3 -m pip install --user pre-commit + fi + fi + + command -v pre-commit >/dev/null 2>&1 || err "pre-commit install failed" + ok "pre-commit installed ($(pre-commit --version))" +} + +install_hooks() { + local root + root="$(git rev-parse --show-toplevel 2>/dev/null)" \ + || err "Not inside a git repository" + cd "$root" + + [[ -f .pre-commit-config.yaml ]] \ + || err "No .pre-commit-config.yaml at $root — create one before installing hooks" + + pre-commit install + pre-commit install --hook-type pre-push + ok "Hooks registered in $root (pre-commit + pre-push)" + + info "Validating configuration..." + pre-commit validate-config + ok "Configuration valid" +} + +verify() { + local root hook target + root="$(git rev-parse --show-toplevel 2>/dev/null)" || err "Not inside a git repository" + + command -v pre-commit >/dev/null 2>&1 \ + && ok "framework: $(pre-commit --version)" \ + || warn "framework: NOT INSTALLED" + + [[ -f "$root/.pre-commit-config.yaml" ]] \ + && ok "config: .pre-commit-config.yaml present" \ + || warn "config: MISSING" + + # A hook that exists but is empty is a silent no-op — treat it as a failure. + # Use --git-common-dir (not "$root/.git") so this resolves correctly from + # inside a worktree too: worktrees share hooks via the main repo's hooks + # dir, and "$root/.git" is a file (not a directory) in a worktree, which + # silently produces false "NOT INSTALLED" reports otherwise. + local git_common_dir + git_common_dir="$(git rev-parse --git-common-dir 2>/dev/null)" || err "Not inside a git repository" + [[ "$git_common_dir" = /* ]] || git_common_dir="$root/$git_common_dir" + for hook in pre-commit pre-push; do + target="$git_common_dir/hooks/$hook" + if [[ ! -f "$target" ]]; then + warn "$hook: NOT INSTALLED" + elif [[ ! -s "$target" ]]; then + warn "$hook: EMPTY (0 bytes) — silent no-op, reports success and checks nothing" + elif [[ ! -x "$target" ]]; then + warn "$hook: not executable" + else + ok "$hook: installed" + fi + done +} + +case "${1:-install}" in + install|"") install_framework; install_hooks ;; + --hooks-only) install_hooks ;; + --verify) verify ;; + *) err "Unknown argument: $1 (use: install | --hooks-only | --verify)" ;; +esac diff --git a/scripts/lib/detect-os.sh b/scripts/lib/detect-os.sh new file mode 100755 index 00000000..77d7f89e --- /dev/null +++ b/scripts/lib/detect-os.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# detect-os.sh — Shared OS/arch detection library +# +# Source this file; do not execute it directly. +# Usage: source "$(dirname "${BASH_SOURCE[0]}")/lib/detect-os.sh" +# +# Exports after sourcing: +# OS — macos | ubuntu | debian | fedora | ... +# OS_VERSION — version string (e.g. "24.04", "14.5") +# PKG_MANAGER — brew | apt | dnf | yum +# ARCH — amd64 | arm64 +# GOOS — darwin | linux (matches Go/tool naming) +# SHELL_RC — absolute path to user shell RC file +# PIP_FLAGS — pip install flags appropriate for the OS +# +# Helper functions (available after sourcing): +# is_macos() — returns 0 if macOS +# is_linux() — returns 0 if Linux +# is_wsl2() — returns 0 if inside WSL2 +# sha256check() — portable SHA256 verification +# http_get() — portable file download (curl-based) + +# ── OS + package manager ────────────────────────────────────────────────────── + +detect_os() { + local kernel + kernel="$(uname -s)" + + case "$kernel" in + Darwin) + OS="macos" + OS_VERSION="$(sw_vers -productVersion 2>/dev/null || echo 'unknown')" + PKG_MANAGER="brew" + GOOS="darwin" + ;; + Linux) + if [[ -f /etc/os-release ]]; then + # shellcheck source=/dev/null + . /etc/os-release + OS="${ID:-linux}" + OS_VERSION="${VERSION_ID:-unknown}" + else + OS="linux" + OS_VERSION="unknown" + fi + GOOS="linux" + + if command -v apt-get &>/dev/null; then + PKG_MANAGER="apt" + elif command -v dnf &>/dev/null; then + PKG_MANAGER="dnf" + elif command -v yum &>/dev/null; then + PKG_MANAGER="yum" + else + echo "[detect-os] ERROR: No supported package manager found (apt/dnf/yum)" >&2 + return 1 + fi + ;; + *) + echo "[detect-os] ERROR: Unsupported kernel: $kernel" >&2 + return 1 + ;; + esac + + export OS OS_VERSION PKG_MANAGER GOOS +} + +# ── CPU architecture ────────────────────────────────────────────────────────── + +detect_arch() { + case "$(uname -m)" in + x86_64) ARCH="amd64" ;; + aarch64|arm64) ARCH="arm64" ;; + *) + echo "[detect-os] ERROR: Unsupported architecture: $(uname -m)" >&2 + return 1 + ;; + esac + export ARCH +} + +# ── Shell RC file ───────────────────────────────────────────────────────────── + +detect_shell_rc() { + if [[ "$GOOS" == "darwin" ]]; then + # macOS defaults to zsh since Catalina + SHELL_RC="$HOME/.zshrc" + else + SHELL_RC="$HOME/.bashrc" + fi + export SHELL_RC +} + +# ── pip install flags ───────────────────────────────────────────────────────── + +detect_pip_flags() { + # Both macOS (Homebrew Python 3.12+) and Ubuntu 22.04+ enforce PEP 668 + PIP_FLAGS="--break-system-packages --user" + export PIP_FLAGS +} + +# ── Helper: portable SHA256 checksum verification ──────────────────────────── +# +# Usage: sha256check +# Returns 0 on match, 1 on mismatch. + +sha256check() { + local archive="$1" checksums="$2" + local basename + basename="$(basename "$archive")" + + # Exact-filename match on the last whitespace-separated field, stripping any + # leading "*" (binary-mode checksum files, e.g. hadolint's checksums.sha256). + # A plain substring grep is unsafe: some releases (e.g. golangci-lint) also list + # derived files like ".tar.gz.sbom.json" whose name is a superstring of the + # binary's, so a substring match pulls in extra lines sha256sum can't find locally. + local line + line="$(awk -v f="$basename" '{n=$NF; sub(/^\*/, "", n); if (n == f) print}' "$checksums" 2>/dev/null || true)" + + # On macOS, always prefer shasum — sha256sum may exist (e.g. from coreutils) but + # the BSD variant doesn't accept GNU flags (--check, --status). + if [[ "$GOOS" == "darwin" ]]; then + if ! command -v shasum &>/dev/null; then + echo "[detect-os] ERROR: shasum not found on macOS" >&2 + return 1 + fi + if [[ -z "$line" ]]; then + # Bare-hash format (e.g. hadolint): single hash on first line, no filename + local hash + hash="$(head -1 "$checksums" | awk '{print $1}')" + [[ -z "$hash" ]] && return 1 + echo "${hash} ${basename}" | ( cd "$(dirname "$archive")" && shasum -a 256 -c --status ) + else + ( cd "$(dirname "$archive")" && echo "$line" | shasum -a 256 -c --status ) + fi + elif command -v sha256sum &>/dev/null; then + if [[ -z "$line" ]]; then + # Bare-hash format: single hash on first line, no filename + local hash + hash="$(head -1 "$checksums" | awk '{print $1}')" + [[ -z "$hash" ]] && return 1 + echo "${hash} ${basename}" | ( cd "$(dirname "$archive")" && sha256sum --check --status ) + else + ( cd "$(dirname "$archive")" && echo "$line" | sha256sum --check --status ) + fi + else + echo "[detect-os] ERROR: No sha256sum or shasum found" >&2 + return 1 + fi +} + +export -f sha256check + +# ── Helper: portable file download ─────────────────────────────────────────── +# +# Usage: http_get + +http_get() { + local url="$1" out="$2" + curl -fsSL --retry 2 "$url" -o "$out" +} + +export -f http_get + +# ── Helper predicates ───────────────────────────────────────────────────────── + +is_macos() { [[ "$GOOS" == "darwin" ]]; } +is_linux() { [[ "$GOOS" == "linux" ]]; } +is_wsl2() { grep -qiE "microsoft|wsl" /proc/version 2>/dev/null; } + +export -f is_macos is_linux is_wsl2 + +# ── Run detection on source ─────────────────────────────────────────────────── + +detect_os +detect_arch +detect_shell_rc +detect_pip_flags diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index a0983689..3d39bea6 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -62,7 +62,7 @@ check_service() { log_info "Waiting for $name to be healthy..." - for i in $(seq 1 $max_retries); do + for _attempt in $(seq 1 "$max_retries"); do response=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$url/health" 2>/dev/null || echo "000") if [ "$response" = "200" ]; then log_success "$name is healthy" diff --git a/squawk-client-go/Makefile b/squawk-client-go/Makefile index 0b246fc9..0fcc9be7 100644 --- a/squawk-client-go/Makefile +++ b/squawk-client-go/Makefile @@ -35,30 +35,30 @@ build: clean build-all: clean @echo "Building $(APP_NAME) for multiple platforms..." @mkdir -p $(BUILD_DIR) - + # Linux @echo "Building for Linux/amd64..." @mkdir -p $(BUILD_DIR)/linux-amd64 CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(GOFLAGS) -o $(BUILD_DIR)/linux-amd64/$(APP_NAME) ./$(SRC_DIR) - + @echo "Building for Linux/arm64..." @mkdir -p $(BUILD_DIR)/linux-arm64 CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GOFLAGS) -o $(BUILD_DIR)/linux-arm64/$(APP_NAME) ./$(SRC_DIR) - + # macOS @echo "Building for macOS/amd64..." @mkdir -p $(BUILD_DIR)/darwin-amd64 CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(GOFLAGS) -o $(BUILD_DIR)/darwin-amd64/$(APP_NAME) ./$(SRC_DIR) - + @echo "Building for macOS/arm64..." @mkdir -p $(BUILD_DIR)/darwin-arm64 CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build $(GOFLAGS) -o $(BUILD_DIR)/darwin-arm64/$(APP_NAME) ./$(SRC_DIR) - + # Windows @echo "Building for Windows/amd64..." @mkdir -p $(BUILD_DIR)/windows-amd64 CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build $(GOFLAGS) -o $(BUILD_DIR)/windows-amd64/$(APP_NAME).exe ./$(SRC_DIR) - + @echo "Build complete. Binaries available in $(BUILD_DIR)/" # Install dependencies @@ -119,18 +119,18 @@ uninstall: package: build-all @echo "Creating release packages..." @mkdir -p $(BUILD_DIR)/packages - + # Linux packages cd $(BUILD_DIR)/linux-amd64 && tar -czf ../packages/$(APP_NAME)-$(VERSION)-linux-amd64.tar.gz $(APP_NAME) cd $(BUILD_DIR)/linux-arm64 && tar -czf ../packages/$(APP_NAME)-$(VERSION)-linux-arm64.tar.gz $(APP_NAME) - - # macOS packages + + # macOS packages cd $(BUILD_DIR)/darwin-amd64 && tar -czf ../packages/$(APP_NAME)-$(VERSION)-darwin-amd64.tar.gz $(APP_NAME) cd $(BUILD_DIR)/darwin-arm64 && tar -czf ../packages/$(APP_NAME)-$(VERSION)-darwin-arm64.tar.gz $(APP_NAME) - + # Windows packages cd $(BUILD_DIR)/windows-amd64 && zip -q ../packages/$(APP_NAME)-$(VERSION)-windows-amd64.zip $(APP_NAME).exe - + @echo "Packages created in $(BUILD_DIR)/packages/" # Run the application in development mode @@ -206,4 +206,4 @@ help: @echo " make build" @echo " make build GOOS=linux GOARCH=amd64" @echo " make run ARGS='-d example.com -s https://dns.google/resolve'" - @echo " make package VERSION=1.2.3" \ No newline at end of file + @echo " make package VERSION=1.2.3" diff --git a/squawk-client-go/README-License.md b/squawk-client-go/README-License.md index b6b41fa1..b86f5188 100644 --- a/squawk-client-go/README-License.md +++ b/squawk-client-go/README-License.md @@ -164,4 +164,4 @@ If you're upgrading from legacy token auth: For license issues: - Visit: https://license.squawkdns.com/portal/login - Sales: Contact your sales representative -- Technical: Check logs with `-v` flag for details \ No newline at end of file +- Technical: Check logs with `-v` flag for details diff --git a/squawk-client-go/README.md b/squawk-client-go/README.md index b1f52c37..7b79f69f 100644 --- a/squawk-client-go/README.md +++ b/squawk-client-go/README.md @@ -524,9 +524,9 @@ client: # Multiple servers with automatic failover server_urls: - "https://192.168.1.100:8443" - - "https://192.168.1.101:8443" + - "https://192.168.1.101:8443" - "https://10.0.0.50:8443" - + # Failover settings max_retries: 6 # Total retry attempts (default: servers * 2) retry_delay: 2 # Seconds between retries (default: 2) @@ -616,4 +616,4 @@ For more information, see the main project documentation. - **Documentation**: [docs.squawkdns.com](https://docs.squawkdns.com) - **GitHub Issues**: [Report Issues](https://github.com/penguintechinc/squawk/issues) -- **Main Project**: [Squawk DNS System](../README.md) \ No newline at end of file +- **Main Project**: [Squawk DNS System](../README.md) diff --git a/squawk-client-go/cmd/squawk-client/main.go b/squawk-client-go/cmd/squawk-client/main.go index 7f378ef7..150fc9e2 100644 --- a/squawk-client-go/cmd/squawk-client/main.go +++ b/squawk-client-go/cmd/squawk-client/main.go @@ -85,26 +85,26 @@ func init() { // Global flags rootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "Configuration file path") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose output") - + // DNS query flags rootCmd.Flags().StringVarP(&domain, "domain", "d", "", "Domain to query (required)") rootCmd.Flags().StringVarP(&recordType, "type", "t", "A", "DNS record type") rootCmd.Flags().BoolVarP(&jsonOutput, "json", "j", false, "Output in JSON format") - + // Server connection flags rootCmd.Flags().StringVarP(&serverURL, "server", "s", "", "DNS server URL") rootCmd.Flags().StringVarP(&authToken, "auth", "a", "", "Authentication token") - + // mTLS flags rootCmd.Flags().StringVar(&clientCert, "client-cert", "", "Client certificate file for mTLS") rootCmd.Flags().StringVar(&clientKey, "client-key", "", "Client private key file for mTLS") rootCmd.Flags().StringVar(&caCert, "ca-cert", "", "CA certificate file for server verification") rootCmd.Flags().BoolVar(&verifySSL, "verify-ssl", true, "Verify SSL/TLS certificates") - + // DNS forwarding flags rootCmd.Flags().BoolVarP(&udpForward, "udp", "u", false, "Enable UDP DNS forwarding on port 53") rootCmd.Flags().BoolVarP(&tcpForward, "tcp", "T", false, "Enable TCP DNS forwarding on port 53") - + // Performance monitoring flags rootCmd.Flags().BoolVar(&enablePerformanceMonitoring, "performance", false, "Enable DNS performance monitoring (Enterprise feature)") @@ -539,7 +539,7 @@ func startMetricsServer(m *metrics.Metrics) { // printDNSResponse prints the DNS response in a human-readable format func printDNSResponse(response *client.DNSResponse) { fmt.Printf("DNS Response Status: %d\n", response.Status) - + if response.Comment != "" { fmt.Printf("Comment: %s\n", response.Comment) } @@ -963,4 +963,4 @@ var timeStatusCmd = &cobra.Command{ } } }, -} \ No newline at end of file +} diff --git a/squawk-client-go/pkg/client/doh_client.go b/squawk-client-go/pkg/client/doh_client.go index 4a2e6c9a..437dfbfb 100644 --- a/squawk-client-go/pkg/client/doh_client.go +++ b/squawk-client-go/pkg/client/doh_client.go @@ -26,7 +26,7 @@ var ( // - Cannot end with a hyphen // - Max 63 characters per label dnsLabelRegex = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?$`) - + // Valid DNS record types validRecordTypes = map[string]bool{ "A": true, @@ -57,21 +57,21 @@ func validateDNSName(domain string) error { if len(domain) > 253 { return fmt.Errorf("DNS name too long: %d characters (max 253)", len(domain)) } - + // Remove trailing dot if present (valid in DNS but we'll validate without it) domain = strings.TrimSuffix(domain, ".") - + // Check for invalid characters at the domain level if strings.ContainsAny(domain, " !@#$%^&*()+={}[]|\\:;\"'<>,?/`~") { return fmt.Errorf("DNS name contains invalid characters") } - + // Split into labels and validate each labels := strings.Split(domain, ".") if len(labels) == 0 { return fmt.Errorf("DNS name has no labels") } - + for i, label := range labels { // Check label length (max 63 characters) if len(label) == 0 { @@ -80,12 +80,12 @@ func validateDNSName(domain string) error { if len(label) > 63 { return fmt.Errorf("DNS label '%s' too long: %d characters (max 63)", label, len(label)) } - + // Special case: TLD can be all numeric for reverse DNS (e.g., "1.0.0.127.in-addr.arpa") if i == len(labels)-1 && label == "arpa" { continue // Skip validation for .arpa TLD } - + // Check label format if !dnsLabelRegex.MatchString(label) { // Special case for IDN/punycode domains @@ -94,14 +94,14 @@ func validateDNSName(domain string) error { } return fmt.Errorf("invalid DNS label '%s': must start/end with alphanumeric and contain only letters, digits, and hyphens", label) } - + // Check for consecutive hyphens (sometimes indicates typos) if strings.Contains(label, "--") && !strings.HasPrefix(label, "xn--") { // Allow -- only in punycode domains return fmt.Errorf("invalid DNS label '%s': contains consecutive hyphens", label) } } - + return nil } @@ -227,7 +227,7 @@ func validateServerURL(serverURL string) error { if strings.ToLower(host) == "localhost" { return nil } - + // Special case: allow well-known public DNS providers to prevent breaking existing configs allowedHosts := []string{ "dns.google", @@ -241,7 +241,7 @@ func validateServerURL(serverURL string) error { "dns.nextdns.io", "doh.cleanbrowsing.org", } - + hostLower := strings.ToLower(host) for _, allowed := range allowedHosts { if hostLower == allowed || strings.HasPrefix(hostLower, allowed + ".") { @@ -253,7 +253,7 @@ func validateServerURL(serverURL string) error { return nil } } - + return fmt.Errorf("server URL must use an IP address (not hostname '%s') to prevent DNS resolution loops. Use the IP address of your DNS server instead", host) } @@ -320,7 +320,7 @@ func NewDoHClient(config *Config) (*DoHClient, error) { func (c *DoHClient) setupHTTPClient() error { tlsConfig := &tls.Config{ // #nosec G402 - InsecureSkipVerify is controlled by verifySSL config option - // When verifySSL is true (default), this becomes false (secure) + // When verifySSL is true (default), this becomes false (secure) // When verifySSL is false (user choice), this becomes true (for testing only) InsecureSkipVerify: !c.verifySSL, } @@ -375,7 +375,7 @@ func (c *DoHClient) Query(ctx context.Context, domain, recordType string) (*DNSR if err := validateDNSName(domain); err != nil { return nil, fmt.Errorf("invalid domain name: %w", err) } - + // Validate and normalize record type if recordType == "" { recordType = "A" @@ -391,7 +391,7 @@ func (c *DoHClient) Query(ctx context.Context, domain, recordType string) (*DNSR // Try each server with retry logic for attempt := 0; attempt < c.maxRetries; attempt++ { serverURL := c.serverURLs[c.currentIndex] - + // Build request URL with query parameters req, err := http.NewRequestWithContext(ctx, "GET", serverURL, nil) if err != nil { @@ -422,7 +422,7 @@ func (c *DoHClient) Query(ctx context.Context, domain, recordType string) (*DNSR lastErr = fmt.Errorf("HTTP request failed for %s: %w", serverURL, err) errors = append(errors, lastErr.Error()) c.nextServer() - + // Add delay before next attempt if attempt < c.maxRetries-1 { select { @@ -473,7 +473,7 @@ func (c *DoHClient) Query(ctx context.Context, domain, recordType string) (*DNSR if len(errors) > 1 { return nil, fmt.Errorf("all DNS servers failed after %d attempts: %s", c.maxRetries, strings.Join(errors, "; ")) } - + return nil, fmt.Errorf("DNS query failed: %w", lastErr) } @@ -488,23 +488,23 @@ func normalizeServerURL(serverURL string) string { if err != nil { return serverURL } - + host := strings.ToLower(parsedURL.Hostname()) - + // Google DNS - ensure correct path if strings.Contains(host, "dns.google") { if parsedURL.Path == "" || parsedURL.Path == "/" { parsedURL.Path = "/resolve" } } - + // Cloudflare DNS - ensure correct path if strings.Contains(host, "cloudflare") || host == "1.1.1.1" || host == "1.0.0.1" { if parsedURL.Path == "" || parsedURL.Path == "/" { parsedURL.Path = "/dns-query" } } - + // Quad9 DNS if strings.Contains(host, "dns.quad9.net") { if parsedURL.Path == "" || parsedURL.Path == "/" { @@ -527,7 +527,7 @@ func (c *DoHClient) QueryWithJSON(ctx context.Context, domain, recordType string if err := validateDNSName(domain); err != nil { return nil, fmt.Errorf("invalid domain name: %w", err) } - + // Validate and normalize record type if recordType == "" { recordType = "A" @@ -614,4 +614,4 @@ func (c *DoHClient) Close() error { c.httpClient.CloseIdleConnections() } return nil -} \ No newline at end of file +} diff --git a/squawk-client-go/pkg/config/config.go b/squawk-client-go/pkg/config/config.go index ed4c7c48..652739ad 100644 --- a/squawk-client-go/pkg/config/config.go +++ b/squawk-client-go/pkg/config/config.go @@ -174,7 +174,7 @@ func loadFromFile(filename string, config *AppConfig) error { if strings.Contains(filename, "..") { return fmt.Errorf("invalid filename: directory traversal not allowed") } - + // #nosec G304 - This reads user-specified config files, validated against directory traversal data, err := os.ReadFile(filename) if err != nil { @@ -209,7 +209,7 @@ func loadFromEnv(config *AppConfig) { if serverURL := os.Getenv("SQUAWK_SERVER_URL"); serverURL != "" { config.Client.ServerURL = serverURL } - + // Multiple server URLs (comma-separated) if serverURLs := os.Getenv("SQUAWK_SERVER_URLS"); serverURLs != "" { urls := strings.Split(serverURLs, ",") @@ -218,14 +218,14 @@ func loadFromEnv(config *AppConfig) { } config.Client.ServerURLs = urls } - + // Retry configuration if maxRetries := os.Getenv("SQUAWK_MAX_RETRIES"); maxRetries != "" { if retries, err := strconv.Atoi(maxRetries); err == nil && retries > 0 { config.Client.MaxRetries = retries } } - + if retryDelay := os.Getenv("SQUAWK_RETRY_DELAY"); retryDelay != "" { if delay, err := strconv.Atoi(retryDelay); err == nil && delay > 0 { config.Client.RetryDelay = delay @@ -241,7 +241,7 @@ func loadFromEnv(config *AppConfig) { if clientCert := os.Getenv("CLIENT_CERT_PATH"); clientCert != "" { config.Client.ClientCert = clientCert } - + if clientKey := os.Getenv("SQUAWK_CLIENT_KEY"); clientKey != "" { config.Client.ClientKey = clientKey } @@ -249,7 +249,7 @@ func loadFromEnv(config *AppConfig) { if clientKey := os.Getenv("CLIENT_KEY_PATH"); clientKey != "" { config.Client.ClientKey = clientKey } - + if caCert := os.Getenv("SQUAWK_CA_CERT"); caCert != "" { config.Client.CaCert = caCert } @@ -257,7 +257,7 @@ func loadFromEnv(config *AppConfig) { if caCert := os.Getenv("CA_CERT_PATH"); caCert != "" { config.Client.CaCert = caCert } - + if verifySSL := os.Getenv("SQUAWK_VERIFY_SSL"); verifySSL != "" { if val, err := strconv.ParseBool(verifySSL); err == nil { config.Client.VerifySSL = val @@ -583,4 +583,4 @@ func maskToken(token string) string { return strings.Repeat("*", len(token)) } return token[:4] + strings.Repeat("*", len(token)-8) + token[len(token)-4:] -} \ No newline at end of file +} diff --git a/squawk-client-go/pkg/forwarder/forwarder.go b/squawk-client-go/pkg/forwarder/forwarder.go index ac1bf14a..23f139ff 100644 --- a/squawk-client-go/pkg/forwarder/forwarder.go +++ b/squawk-client-go/pkg/forwarder/forwarder.go @@ -334,4 +334,4 @@ func (f *Forwarder) IsRunning() bool { f.mu.RLock() defer f.mu.RUnlock() return f.running -} \ No newline at end of file +} diff --git a/squawk-client-go/pkg/license/validator.go b/squawk-client-go/pkg/license/validator.go index 40d65a02..10171841 100644 --- a/squawk-client-go/pkg/license/validator.go +++ b/squawk-client-go/pkg/license/validator.go @@ -67,7 +67,7 @@ func (v *Validator) ValidateLicense(ctx context.Context) (*ValidationResponse, e } today := time.Now().Format("2006-01-02") - + // Check if we've already validated today v.cacheMutex.RLock() if v.validatedToday == today { @@ -146,7 +146,7 @@ func (v *Validator) validateLicenseKey(ctx context.Context) (*ValidationResponse v.cacheValidation("license_validation", validationResp.Valid, validationResp.Message) v.lastValidate = time.Now() - + // Mark as validated today v.cacheMutex.Lock() v.validatedToday = time.Now().Format("2006-01-02") @@ -181,7 +181,7 @@ func (v *Validator) validateUserToken(ctx context.Context) (*ValidationResponse, v.cacheValidation("token_validation", validationResp.Valid, validationResp.Message) v.lastValidate = time.Now() - + // Mark as validated today v.cacheMutex.Lock() v.validatedToday = time.Now().Format("2006-01-02") @@ -206,7 +206,7 @@ func (v *Validator) cacheValidation(key string, valid bool, message string) { // IsValid returns true if the current license/token is valid func (v *Validator) IsValid(ctx context.Context) (bool, error) { today := time.Now().Format("2006-01-02") - + // Check if we've already validated today - use that result v.cacheMutex.RLock() if v.validatedToday == today { @@ -280,4 +280,4 @@ func (v *Validator) GetLicenseInfo(ctx context.Context) (string, error) { } return info, nil -} \ No newline at end of file +} diff --git a/squawk-client-go/pkg/logger/logger.go b/squawk-client-go/pkg/logger/logger.go index 3ed155fd..69328663 100644 --- a/squawk-client-go/pkg/logger/logger.go +++ b/squawk-client-go/pkg/logger/logger.go @@ -48,4 +48,4 @@ func (l *SimpleLogger) Error(format string, args ...interface{}) { // Printf logs a formatted message func (l *SimpleLogger) Printf(format string, args ...interface{}) { fmt.Printf(format, args...) -} \ No newline at end of file +} diff --git a/squawk-client-go/pkg/performance/dns_performance.go b/squawk-client-go/pkg/performance/dns_performance.go index c237cf81..e5314ae1 100644 --- a/squawk-client-go/pkg/performance/dns_performance.go +++ b/squawk-client-go/pkg/performance/dns_performance.go @@ -29,42 +29,42 @@ type DNSPerformanceStats struct { ServerURL string `json:"server_url"` TestDomain string `json:"test_domain"` QueryType string `json:"query_type"` - + // Network Timing (similar to http-traceroute) DNSLookup Duration `json:"dns_lookup"` // DNS resolution time - TCPConnection Duration `json:"tcp_connection"` // TCP connect time + TCPConnection Duration `json:"tcp_connection"` // TCP connect time TLSHandshake Duration `json:"tls_handshake"` // TLS handshake time ServerProcessing Duration `json:"server_processing"` // Time to first byte ContentTransfer Duration `json:"content_transfer"` // Content download time - + // Total Times TotalTime Duration `json:"total_time"` // End-to-end total time NameLookup Duration `json:"name_lookup"` // DNS + TCP + TLS Connect Duration `json:"connect"` // TCP + TLS - + // HTTP Details HTTPStatus int `json:"http_status"` HTTPHeaders int `json:"http_headers_size"` ResponseSize int64 `json:"response_size"` - + // DNS Response Details DNSStatus string `json:"dns_status"` // NOERROR, NXDOMAIN, etc. DNSAnswerCount int `json:"dns_answer_count"` DNSResponseCode int `json:"dns_response_code"` CacheHit bool `json:"cache_hit"` - + // Network Information LocalAddr string `json:"local_addr"` RemoteAddr string `json:"remote_addr"` Protocol string `json:"protocol"` // HTTP/1.1, HTTP/2, etc. TLSVersion string `json:"tls_version"` TLSCipherSuite string `json:"tls_cipher_suite"` - + // Error Information ErrorType string `json:"error_type,omitempty"` ErrorMessage string `json:"error_message,omitempty"` Successful bool `json:"successful"` - + // Performance Metrics Jitter Duration `json:"jitter,omitempty"` // Compared to baseline PacketLoss float64 `json:"packet_loss,omitempty"` // If detectable @@ -89,11 +89,11 @@ func (d *Duration) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(data, &temp); err != nil { return err } - + if ns, ok := temp["nanoseconds"].(float64); ok { d.Duration = time.Duration(int64(ns)) } - + return nil } @@ -104,15 +104,15 @@ type DNSPerformanceMonitor struct { logger logger.Logger stopChan chan struct{} wg sync.WaitGroup - + // Performance tracking baseline map[string]Duration // Baseline response times per domain recentStats []DNSPerformanceStats statsMutex sync.RWMutex - + // Test domains for performance monitoring testDomains []string - + // Upload configuration uploadURL string uploadInterval time.Duration @@ -131,7 +131,7 @@ func NewDNSPerformanceMonitor(cfg *client.Config, log logger.Logger) *DNSPerform MaxIdleConns: 10, IdleConnTimeout: 30 * time.Second, } - + // Load client certificates if provided if cfg.ClientCert != "" && cfg.ClientKey != "" { cert, err := tls.LoadX509KeyPair(cfg.ClientCert, cfg.ClientKey) @@ -139,7 +139,7 @@ func NewDNSPerformanceMonitor(cfg *client.Config, log logger.Logger) *DNSPerform transport.TLSClientConfig.Certificates = []tls.Certificate{cert} } } - + // Load CA certificate if provided if cfg.CaCert != "" { caCert, err := os.ReadFile(cfg.CaCert) @@ -149,27 +149,27 @@ func NewDNSPerformanceMonitor(cfg *client.Config, log logger.Logger) *DNSPerform transport.TLSClientConfig.RootCAs = caCertPool } } - + client := &http.Client{ Transport: transport, Timeout: 30 * time.Second, // Default timeout } - + // Default test domains for performance monitoring testDomains := []string{ "google.com", - "cloudflare.com", + "cloudflare.com", "example.com", "github.com", cfg.ServerURL, // Include the DNS server itself } - + uploadURL := cfg.ServerURL if !strings.HasSuffix(uploadURL, "/") { uploadURL += "/" } uploadURL += "api/performance/upload" - + return &DNSPerformanceMonitor{ config: cfg, client: client, @@ -187,55 +187,55 @@ func NewDNSPerformanceMonitor(cfg *client.Config, log logger.Logger) *DNSPerform // Start begins performance monitoring func (pm *DNSPerformanceMonitor) Start() error { pm.logger.Info("Starting DNS performance monitoring") - + pm.wg.Add(2) - + // Start performance testing goroutine go pm.performanceTestLoop() - - // Start upload goroutine + + // Start upload goroutine go pm.uploadLoop() - + return nil } // Stop stops performance monitoring func (pm *DNSPerformanceMonitor) Stop() error { pm.logger.Info("Stopping DNS performance monitoring") - + close(pm.stopChan) pm.wg.Wait() - + // Upload any remaining stats pm.uploadStats() - + return nil } // performanceTestLoop runs performance tests at random intervals func (pm *DNSPerformanceMonitor) performanceTestLoop() { defer pm.wg.Done() - + ticker := time.NewTicker(time.Minute) // Check every minute if we should test defer ticker.Stop() - + for { select { case <-pm.stopChan: return - + case <-ticker.C: // Random interval between 5 and 10 minutes // #nosec G404 - Using math/rand is acceptable for test scheduling (non-security context) nextTest := rand.Intn(5*60) + 5*60 // 5-10 minutes in seconds - + timer := time.NewTimer(time.Duration(nextTest) * time.Second) - + select { case <-pm.stopChan: timer.Stop() return - + case <-timer.C: pm.runPerformanceTest() } @@ -248,25 +248,25 @@ func (pm *DNSPerformanceMonitor) runPerformanceTest() { // Select random test domain // #nosec G404 - Using math/rand is acceptable for domain selection (non-security context) domain := pm.testDomains[rand.Intn(len(pm.testDomains))] - + pm.logger.Debug("Running performance test for domain: %s", domain) - + stats := pm.performDNSOverHTTPTest(domain, "A") - + pm.statsMutex.Lock() pm.recentStats = append(pm.recentStats, stats) - + // Keep only recent stats (last 100) if len(pm.recentStats) > 100 { pm.recentStats = pm.recentStats[len(pm.recentStats)-100:] } pm.statsMutex.Unlock() - + // Update baseline if successful if stats.Successful { pm.updateBaseline(domain, stats.TotalTime) } - + pm.logger.Debug("Performance test completed: %s in %v", domain, stats.TotalTime.Duration) } @@ -281,7 +281,7 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string Protocol: "HTTP/1.1", // Will be updated based on actual connection Successful: false, } - + // Create DNS over HTTP request URL dnsURL, err := url.Parse(pm.config.ServerURL) if err != nil { @@ -289,13 +289,13 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string stats.ErrorMessage = err.Error() return stats } - + // Add DNS query parameters params := url.Values{} params.Set("name", domain) params.Set("type", queryType) dnsURL.RawQuery = params.Encode() - + // Create request with tracing req, err := http.NewRequest("GET", dnsURL.String(), nil) if err != nil { @@ -303,15 +303,15 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string stats.ErrorMessage = err.Error() return stats } - + // Set headers req.Header.Set("Accept", "application/dns-json") req.Header.Set("User-Agent", "Squawk-DNS-Client/2.0 Performance-Monitor") - + if pm.config.AuthToken != "" { req.Header.Set("Authorization", "Bearer "+pm.config.AuthToken) } - + // Setup request tracing var ( dnsStart, dnsEnd time.Time @@ -321,7 +321,7 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string firstByteTime time.Time localAddr, remoteAddr string ) - + trace := &httptrace.ClientTrace{ DNSStart: func(info httptrace.DNSStartInfo) { dnsStart = time.Now() @@ -344,7 +344,7 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string TLSHandshakeDone: func(state tls.ConnectionState, err error) { tlsEnd = time.Now() stats.TLSHandshake = Duration{tlsEnd.Sub(tlsStart)} - + if err == nil { stats.TLSVersion = pm.tlsVersionString(state.Version) stats.TLSCipherSuite = tls.CipherSuiteName(state.CipherSuite) @@ -366,17 +366,17 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string } }, } - + req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) - + // Perform the request reqStart = time.Now() resp, err := pm.client.Do(req) reqEnd = time.Now() - + stats.LocalAddr = localAddr stats.RemoteAddr = remoteAddr - + if err != nil { stats.ErrorType = "http_request_error" stats.ErrorMessage = err.Error() @@ -384,32 +384,32 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string return stats } defer func() { _ = resp.Body.Close() }() - + // Read response body, err := io.ReadAll(resp.Body) contentEnd := time.Now() - + stats.HTTPStatus = resp.StatusCode stats.ResponseSize = int64(len(body)) stats.HTTPHeaders = pm.calculateHeadersSize(resp.Header) - + // Calculate timing metrics stats.TotalTime = Duration{contentEnd.Sub(reqStart)} - + if !firstByteTime.IsZero() { stats.ServerProcessing = Duration{firstByteTime.Sub(reqStart)} stats.ContentTransfer = Duration{contentEnd.Sub(firstByteTime)} } - + stats.NameLookup = Duration{stats.DNSLookup.Duration + stats.TCPConnection.Duration + stats.TLSHandshake.Duration} stats.Connect = Duration{stats.TCPConnection.Duration + stats.TLSHandshake.Duration} - + if err != nil { stats.ErrorType = "response_read_error" stats.ErrorMessage = err.Error() return stats } - + // Parse DNS response if successful if resp.StatusCode == 200 { pm.parseDNSResponse(body, &stats) @@ -418,7 +418,7 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string stats.ErrorType = "http_error" stats.ErrorMessage = fmt.Sprintf("HTTP %d", resp.StatusCode) } - + // Calculate jitter if we have baseline if baseline, exists := pm.baseline[domain]; exists { jitter := stats.TotalTime.Duration - baseline.Duration @@ -427,7 +427,7 @@ func (pm *DNSPerformanceMonitor) performDNSOverHTTPTest(domain, queryType string } stats.Jitter = Duration{jitter} } - + return stats } @@ -438,7 +438,7 @@ func (pm *DNSPerformanceMonitor) parseDNSResponse(body []byte, stats *DNSPerform stats.DNSStatus = "PARSE_ERROR" return } - + // Extract DNS response code if status, ok := response["Status"].(float64); ok { stats.DNSResponseCode = int(status) @@ -451,12 +451,12 @@ func (pm *DNSPerformanceMonitor) parseDNSResponse(body []byte, stats *DNSPerform stats.DNSStatus = fmt.Sprintf("RCODE_%d", int(status)) } } - + // Extract answer count if answers, ok := response["Answer"].([]interface{}); ok { stats.DNSAnswerCount = len(answers) } - + // Check for cache hit indicator if comment, ok := response["Comment"].(string); ok { stats.CacheHit = strings.Contains(strings.ToLower(comment), "cache") @@ -467,7 +467,7 @@ func (pm *DNSPerformanceMonitor) parseDNSResponse(body []byte, stats *DNSPerform func (pm *DNSPerformanceMonitor) updateBaseline(domain string, responseTime Duration) { pm.statsMutex.Lock() defer pm.statsMutex.Unlock() - + if existing, exists := pm.baseline[domain]; exists { // Use exponential moving average: new_baseline = 0.8 * old + 0.2 * new newTime := time.Duration(float64(existing.Duration)*0.8 + float64(responseTime.Duration)*0.2) @@ -480,15 +480,15 @@ func (pm *DNSPerformanceMonitor) updateBaseline(domain string, responseTime Dura // uploadLoop handles periodic upload of performance stats func (pm *DNSPerformanceMonitor) uploadLoop() { defer pm.wg.Done() - + ticker := time.NewTicker(pm.uploadInterval) defer ticker.Stop() - + for { select { case <-pm.stopChan: return - + case <-ticker.C: pm.uploadStats() } @@ -502,22 +502,22 @@ func (pm *DNSPerformanceMonitor) uploadStats() { pm.statsMutex.Unlock() return } - + // Take up to uploadBatchSize stats batchSize := len(pm.recentStats) if batchSize > pm.uploadBatchSize { batchSize = pm.uploadBatchSize } - + statsToUpload := make([]DNSPerformanceStats, batchSize) copy(statsToUpload, pm.recentStats[:batchSize]) - + // Remove uploaded stats pm.recentStats = pm.recentStats[batchSize:] pm.statsMutex.Unlock() - + pm.logger.Debug("Uploading %d performance statistics", len(statsToUpload)) - + // Create upload payload payload := map[string]interface{}{ "client_id": pm.generateClientID(), @@ -525,45 +525,45 @@ func (pm *DNSPerformanceMonitor) uploadStats() { "stats_count": len(statsToUpload), "statistics": statsToUpload, } - + jsonData, err := json.Marshal(payload) if err != nil { pm.logger.Error("Failed to marshal performance stats: %v", err) return } - + // Create upload request req, err := http.NewRequest("POST", pm.uploadURL, strings.NewReader(string(jsonData))) if err != nil { pm.logger.Error("Failed to create upload request: %v", err) return } - + req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "Squawk-DNS-Client/2.0 Performance-Monitor") - + if pm.config.AuthToken != "" { req.Header.Set("Authorization", "Bearer "+pm.config.AuthToken) } - + // Perform upload ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - + req = req.WithContext(ctx) - + resp, err := pm.client.Do(req) if err != nil { pm.logger.Error("Failed to upload performance stats: %v", err) return } defer func() { _ = resp.Body.Close() }() - + if resp.StatusCode != 200 { pm.logger.Error("Performance stats upload failed with status: %d", resp.StatusCode) return } - + pm.logger.Debug("Successfully uploaded %d performance statistics", len(statsToUpload)) } @@ -573,7 +573,7 @@ func (pm *DNSPerformanceMonitor) generateClientID() string { if err != nil || hostname == "" { hostname = "unknown" } - + // Include hostname and a hash of server URL for uniqueness h := sha256.Sum256([]byte(pm.config.ServerURL + hostname)) return fmt.Sprintf("%s-%x", hostname, h[:8]) @@ -610,11 +610,11 @@ func (pm *DNSPerformanceMonitor) calculateHeadersSize(headers http.Header) int { func (pm *DNSPerformanceMonitor) GetRecentStats() []DNSPerformanceStats { pm.statsMutex.RLock() defer pm.statsMutex.RUnlock() - + // Return a copy to avoid race conditions stats := make([]DNSPerformanceStats, len(pm.recentStats)) copy(stats, pm.recentStats) - + return stats } @@ -622,12 +622,12 @@ func (pm *DNSPerformanceMonitor) GetRecentStats() []DNSPerformanceStats { func (pm *DNSPerformanceMonitor) GetBaselines() map[string]Duration { pm.statsMutex.RLock() defer pm.statsMutex.RUnlock() - + // Return a copy baselines := make(map[string]Duration) for k, v := range pm.baseline { baselines[k] = v } - + return baselines -} \ No newline at end of file +} diff --git a/squawk-client/Dockerfile b/squawk-client/Dockerfile index b414732c..a9965357 100644 --- a/squawk-client/Dockerfile +++ b/squawk-client/Dockerfile @@ -84,4 +84,4 @@ ENV SQUAWK_SERVER_URL=https://dns.google/resolve \ LOG_LEVEL=INFO # Default command - DNS forwarder mode -CMD ["sh", "-c", "python3 /app/dns-client/bins/client.py -s ${SQUAWK_SERVER_URL} -a ${SQUAWK_AUTH_TOKEN} -u -T -v"] \ No newline at end of file +CMD ["sh", "-c", "python3 /app/dns-client/bins/client.py -s ${SQUAWK_SERVER_URL} -a ${SQUAWK_AUTH_TOKEN} -u -T -v"] diff --git a/squawk-client/bins/client.py b/squawk-client/bins/client.py index 61e5cbf2..031e6ace 100755 --- a/squawk-client/bins/client.py +++ b/squawk-client/bins/client.py @@ -137,7 +137,7 @@ def _validate_server_url(dns_server_url): try: parsed_url = urlparse(dns_server_url) except Exception as e: - raise ValueError(f"Invalid DNS server URL format: {e}") + raise ValueError(f"Invalid DNS server URL format: {e}") from e if parsed_url.scheme not in ["http", "https"]: raise ValueError(f"DNS server URL must use http or https scheme, got: {parsed_url.scheme}") @@ -234,7 +234,7 @@ def __init__( self._validate_server_url(url) normalized_urls.append(self._normalize_server_url(url)) except ValueError as e: - raise ValueError(f"Invalid server URL at index {i}: {e}") + raise ValueError(f"Invalid server URL at index {i}: {e}") from e self.dns_server_urls = normalized_urls # Legacy support @@ -471,7 +471,7 @@ def _query_grpc(self, domain, record_type="A"): return self._convert_grpc_response(response) except grpc.RpcError as e: logging.error(f"gRPC query failed: {e.code()} - {e.details()}") - raise Exception(f"gRPC query failed: {e.details()}") + raise Exception(f"gRPC query failed: {e.details()}") from e except Exception as e: logging.error(f"Unexpected error in gRPC query: {e}") raise @@ -510,7 +510,7 @@ def batch_query(self, domains, record_type="A", max_concurrent=10): return [self._convert_grpc_response(r) for r in response.responses] except grpc.RpcError as e: logging.error(f"gRPC batch query failed: {e.code()} - {e.details()}") - raise Exception(f"gRPC batch query failed: {e.details()}") + raise Exception(f"gRPC batch query failed: {e.details()}") from e def health_check(self): """Check if DNS server is healthy""" @@ -638,7 +638,7 @@ def start_tcp_server(self): def handle_request(self, data): domain = "example.com" # Extract the domain from the DNS request record_type = "A" # Extract the record type from the DNS request - result = self.dns_client.query(domain, record_type) + self.dns_client.query(domain, record_type) response = b"" # Create a proper DNS response return response @@ -801,7 +801,7 @@ def main(argv): results = client.batch_query(domains, record_type) else: results = [client.query(d, record_type) for d in domains] - for domain_name, result in zip(domains, results): + for domain_name, result in zip(domains, results, strict=True): print(f"{domain_name}: {json.dumps(result, indent=2)}") else: logging.error(f"Batch file not found: {batch_domains}") diff --git a/squawk-client/bins/k8s-client.py b/squawk-client/bins/k8s-client.py index f661c530..3425d0b8 100644 --- a/squawk-client/bins/k8s-client.py +++ b/squawk-client/bins/k8s-client.py @@ -16,7 +16,7 @@ def resolve(self, name): "Authorization": f"Bearer {self.token}", } params = {"name": name, "type": "A"} - response = requests.get(self.doh_url, headers=headers, params=params) + response = requests.get(self.doh_url, headers=headers, params=params, timeout=10) response.raise_for_status() return response.json() diff --git a/squawk-client/bins/systray.py b/squawk-client/bins/systray.py index 2a844c74..59d6649c 100644 --- a/squawk-client/bins/systray.py +++ b/squawk-client/bins/systray.py @@ -266,7 +266,7 @@ def show_status(self, icon=None, item=None): message += f"\nFailures: {failures}" if self.dns_fallback_active: - message += f"\nFallback: Active (using original DNS)" + message += "\nFallback: Active (using original DNS)" if self.original_dns_servers: message += f"\nOriginal DNS: {', '.join(self.original_dns_servers)}" diff --git a/squawk-client/docker-compose.yml b/squawk-client/docker-compose.yml index 2c955467..44f2ca1c 100644 --- a/squawk-client/docker-compose.yml +++ b/squawk-client/docker-compose.yml @@ -28,18 +28,18 @@ services: # DNS Server Configuration - SQUAWK_SERVER_URL=https://dns.yourdomain.com:8443 - SQUAWK_AUTH_TOKEN=your-secure-token-here - + # Client Configuration - LOG_LEVEL=INFO - SQUAWK_DOMAIN= - SQUAWK_RECORD_TYPE=A - + # mTLS Configuration (optional) - SQUAWK_CLIENT_CERT=/app/certs/client.crt - SQUAWK_CLIENT_KEY=/app/certs/client.key - SQUAWK_CA_CERT=/app/certs/ca.crt - SQUAWK_VERIFY_SSL=true - + # Console URL (optional) - SQUAWK_CONSOLE_URL=http://localhost:8080/health volumes: @@ -72,4 +72,4 @@ services: - /tmp/.X11-unix:/tmp/.X11-unix:rw network_mode: host profiles: - - desktop # Only start with --profile desktop \ No newline at end of file + - desktop # Only start with --profile desktop diff --git a/squawk-client/requirements-dev.txt b/squawk-client/requirements-dev.txt index 3ce4ccba..61e6573b 100644 --- a/squawk-client/requirements-dev.txt +++ b/squawk-client/requirements-dev.txt @@ -2,7 +2,5 @@ pytest>=7.4.3 pytest-cov>=4.1.0 pytest-mock>=3.12.0 -black>=23.9.1 -flake8>=6.1.0 -flake8-bugbear>=24.2.6 +ruff>=0.8.4 mypy>=1.6.1 diff --git a/squawk-client/tests/test_client.py b/squawk-client/tests/test_client.py index a49dd5c4..62d6eb1e 100644 --- a/squawk-client/tests/test_client.py +++ b/squawk-client/tests/test_client.py @@ -17,7 +17,7 @@ class TestDNSOverHTTPSClient: """Test DNS-over-HTTPS client""" - + @patch('client.requests.Session') def test_client_initialization(self, mock_session): """Test client initialization""" @@ -25,10 +25,10 @@ def test_client_initialization(self, mock_session): dns_server_url='https://dns.google/resolve', auth_token='test-token' ) - + assert client.dns_server_url == 'https://dns.google/resolve' assert client.auth_token == 'test-token' - + @patch('client.requests.Session') def test_query_success(self, mock_session): """Test successful DNS query""" @@ -39,39 +39,39 @@ def test_query_success(self, mock_session): 'Status': 0, 'Answer': [{'name': 'example.com', 'type': 1, 'data': '93.184.216.34'}] } - + mock_session_instance = Mock() mock_session_instance.get.return_value = mock_response mock_session.return_value = mock_session_instance - + client = DNSOverHTTPSClient(dns_server_url='https://dns.google/resolve') result = client.query('example.com', 'A') - + assert result['Status'] == 0 assert len(result['Answer']) > 0 - + @patch('client.requests.Session') def test_query_with_auth_token(self, mock_session): """Test DNS query with authentication token""" mock_response = Mock() mock_response.status_code = 200 mock_response.json.return_value = {'Status': 0, 'Answer': []} - + mock_session_instance = Mock() mock_session_instance.get.return_value = mock_response mock_session.return_value = mock_session_instance - + client = DNSOverHTTPSClient( dns_server_url='https://dns.google/resolve', auth_token='test-token-123' ) client.query('example.com', 'A') - + # Verify auth header was sent call_args = mock_session_instance.get.call_args assert 'headers' in call_args[1] assert 'Authorization' in call_args[1]['headers'] - + @patch('client.requests.Session') def test_query_failover(self, mock_session): """Test failover to secondary server""" @@ -99,31 +99,31 @@ def test_query_failover(self, mock_session): result = client.query('example.com', 'A') assert result['Status'] == 0 - + def test_domain_validation(self): """Test domain name validation""" # Valid domains assert DNSOverHTTPSClient.validate_dns_name('example.com') == True assert DNSOverHTTPSClient.validate_dns_name('sub.example.com') == True assert DNSOverHTTPSClient.validate_dns_name('example.co.uk') == True - + # Invalid domains with pytest.raises(ValueError): DNSOverHTTPSClient.validate_dns_name('') - + with pytest.raises(ValueError): DNSOverHTTPSClient.validate_dns_name('invalid domain.com') - + with pytest.raises(ValueError): DNSOverHTTPSClient.validate_dns_name('a' * 300) # Too long - + def test_record_type_validation(self): """Test DNS record type validation""" # Valid types assert DNSOverHTTPSClient.validate_record_type('A') == 'A' assert DNSOverHTTPSClient.validate_record_type('AAAA') == 'AAAA' assert DNSOverHTTPSClient.validate_record_type('mx') == 'MX' # Case insensitive - + # Invalid type with pytest.raises(ValueError): DNSOverHTTPSClient.validate_record_type('INVALID') @@ -229,7 +229,7 @@ def test_grpc_health_check(self, mock_stub, mock_channel, mock_health_request): class TestDNSForwarder: """Test DNS forwarder""" - + @patch('client.DNSOverHTTPSClient') def test_forwarder_initialization(self, mock_client): """Test DNS forwarder initialization""" @@ -240,7 +240,7 @@ def test_forwarder_initialization(self, mock_client): listen_udp=True, listen_tcp=False ) - + assert forwarder.udp_port == 5353 assert forwarder.tcp_port == 5353 assert forwarder.listen_udp == True @@ -249,38 +249,38 @@ def test_forwarder_initialization(self, mock_client): class TestClientIntegration: """Integration tests for client operations""" - + @patch('client.requests.Session') def test_multiple_queries(self, mock_session): """Test multiple sequential queries""" mock_response = Mock() mock_response.status_code = 200 mock_response.json.return_value = {'Status': 0, 'Answer': []} - + mock_session_instance = Mock() mock_session_instance.get.return_value = mock_response mock_session.return_value = mock_session_instance - + client = DNSOverHTTPSClient(dns_server_url='https://dns.google/resolve') - + domains = ['example1.com', 'example2.com', 'example3.com'] for domain in domains: result = client.query(domain, 'A') assert result['Status'] == 0 - + @patch('client.requests.Session') def test_different_record_types(self, mock_session): """Test queries for different record types""" mock_response = Mock() mock_response.status_code = 200 mock_response.json.return_value = {'Status': 0, 'Answer': []} - + mock_session_instance = Mock() mock_session_instance.get.return_value = mock_response mock_session.return_value = mock_session_instance - + client = DNSOverHTTPSClient(dns_server_url='https://dns.google/resolve') - + record_types = ['A', 'AAAA', 'MX', 'TXT', 'CNAME'] for record_type in record_types: result = client.query('example.com', record_type) diff --git a/squawk-client/web/apps/_default/static/index.html b/squawk-client/web/apps/_default/static/index.html index b3df0406..07046883 100644 --- a/squawk-client/web/apps/_default/static/index.html +++ b/squawk-client/web/apps/_default/static/index.html @@ -1,6 +1,6 @@ - +