{ "schema_version": "codestax-public-evidence/v1", "kind": "recorded_live_engine_demonstration", "recorded_at": "2026-11-05T19:37:50.646057+00:00", "scope": "CodeStax-authored controlled sample. Direct calls to the deployed PR AI analysis function, not a customer PR or an end-to-end webhook/queue/database/scanner/provider-delivery run. No repository-wide context was provided.", "engine_function": "tasks_pr._analyze_pr_with_ai", "deployed_source_sha256": { "tasks_pr.py": "0b373507c348cbe1c03c821a1b38b8c8957981e908f360e1bd28594b720412ca", "ai/llm.py": "e06eb4521746c4b5c8a10c9f8cff70da2609040381c0aaacdb9dd8303c069f7b" }, "sources": [ { "artifact": "evaluation/pr_review/walkthrough/2026-11-06/live-engine-capture.json", "sha256": "52970298c7bea4fc06341aaa936d2163d9dbc9d7f5560aff910f89612f282460" }, { "artifact": "evaluation/pr_review/walkthrough/2026-11-06/unavailable-engine-capture.json", "sha256": "ebd4248717b658753afd9c400a14f16688187e287dd4fd3f966084dd87c97968" }, { "artifact": "evaluation/pr_review/walkthrough/2026-11-06/before.py", "sha256": "96d4e1fc41f9eff43f9bd9bfa391c86e1391895233f10d800f5280f24d228809" }, { "artifact": "evaluation/pr_review/walkthrough/2026-11-06/after.py", "sha256": "7e84792a66811f77da21e3777063597609c4c45093c2b7f18026e3cf6635833a" }, { "artifact": "evaluation/pr_review/walkthrough/2026-11-06/validation.json", "sha256": "4c4a288e2d103b7a3115cfa802cce3d7f2417b3d49b41b4b4647d355216fc009" }, { "artifact": "evaluation/pr_review/walkthrough/2026-11-06/capture-live-demo.py", "sha256": "634dbe153bb48b22d3b0fb2bdf98d7d87108cba3ef19afd3f943d0aa801a2a88" }, { "artifact": "scripts/verify-ai-review-walkthrough.py", "sha256": "70aefbc43b34db18afba7321996cb31343965c578e8e4eac9a8b9bed2508cc69" } ], "samples": { "before": "def find_users(connection, query_args):\n \"\"\"Look up users from a request query-parameter mapping.\"\"\"\n email = query_args.get(\"email\", \"\")\n query = f\"SELECT name FROM users WHERE email = '{email}'\"\n return connection.execute(query).fetchall()\n", "after": "def find_users(connection, query_args):\n \"\"\"Look up users from a request query-parameter mapping.\"\"\"\n email = query_args.get(\"email\", \"\")\n if not isinstance(email, str):\n raise ValueError(\"email must be a string\")\n return connection.execute(\n \"SELECT name FROM users WHERE email = ?\", (email,)\n ).fetchall()\n" }, "runs": [ { "label": "before", "analysis_run_id": "public-walkthrough-0fbce53a-6e8f-475c-b842-e51fe2f25011", "recorded_at": "2026-11-05T19:38:18.978203+00:00", "provider": "anthropic", "model_id": "claude-sonnet-4-6", "model_revision": "unversioned_alias", "prompt_version": "v4", "cache_hit": false, "ai_health": "complete", "output_validation": { "accepted": 4, "rejected": 0, "rejections": [] }, "coverage": { "complete": false, "files_total": 1, "files_analyzed": 1, "files_omitted": 1, "omissions": [ { "file": "example.py", "reason": "repository_context_unavailable" } ] }, "patch": "--- a/example.py\n+++ b/example.py\n@@ -0,0 +1,5 @@\n+def find_users(connection, query_args):\n+ \"\"\"Look up users from a request query-parameter mapping.\"\"\"\n+ email = query_args.get(\"email\", \"\")\n+ query = f\"SELECT name FROM users WHERE email = '{email}'\"\n+ return connection.execute(query).fetchall()\n", "accepted_findings": [ { "file": "example.py", "line": 4, "end_line": 4, "severity": "critical", "category": "security", "description": "SQL injection via f-string interpolation: `email` is taken directly from user-supplied `query_args` and embedded into a raw SQL query string without parameterization or escaping. An attacker can supply a value such as `' OR '1'='1` to bypass filtering, or `'; DROP TABLE users; --` to cause destructive operations. This is a complete database compromise vector.", "suggestion": "Use parameterized queries (prepared statements) instead of string interpolation. Example:\n```python\nquery = \"SELECT name FROM users WHERE email = ?\"\nreturn connection.execute(query, (email,)).fetchall()\n```\nFor libraries using `%s` style (e.g., psycopg2):\n```python\nquery = \"SELECT name FROM users WHERE email = %s\"\nreturn connection.execute(query, (email,)).fetchall()\n```\nNever construct SQL strings from user input.", "confidence": 0.99, "rule_id": "sql-injection-string-interpolation", "source": "ai", "source_engine": "Anthropic PR Review" }, { "file": "example.py", "line": 3, "end_line": 4, "severity": "medium", "category": "data_handling", "description": "The function returns raw PII (user names) from the database with no access control check, rate limiting, or result-set size cap. A caller could inadvertently expose all user names if the email filter is bypassed (e.g., via SQL injection) or if an empty string is passed, potentially returning the entire users table.", "suggestion": "Add an explicit check that `email` is non-empty and matches a valid email format before executing the query. Also consider limiting result set size with `LIMIT` in the query and enforcing authorization checks before calling this function.", "confidence": 0.75, "rule_id": "pii-unguarded-return", "source": "ai", "source_engine": "Anthropic PR Review" }, { "file": "example.py", "line": 5, "end_line": 5, "severity": "low", "category": "code_quality", "description": "No error handling around `connection.execute()` or `fetchall()`. Database errors (connection loss, query failure) will propagate as unhandled exceptions to callers, potentially leaking internal database error messages or stack traces to end users.", "suggestion": "Wrap the database call in a try/except block and handle or log exceptions appropriately:\n```python\ntry:\n return connection.execute(query, (email,)).fetchall()\nexcept Exception as exc:\n logger.error(\"Database query failed: %s\", exc)\n raise\n```", "confidence": 0.8, "rule_id": "missing-exception-handling", "source": "ai", "source_engine": "Anthropic PR Review" }, { "file": "example.py", "line": 1, "end_line": 5, "severity": "medium", "category": "architecture", "description": "Raw SQL construction is embedded directly in what appears to be a service/business-logic layer function. There is no input validation at the boundary, no use of an ORM or query builder, and the function accepts an untyped mapping (`query_args`) with no schema enforcement, making it easy to misuse from HTTP handlers.", "suggestion": "Introduce a data-access layer that encapsulates all SQL. Validate and type-check inputs at the boundary (e.g., verify `email` matches a regex before use). Consider using an ORM (SQLAlchemy, Django ORM) or at minimum a query-builder to prevent raw string construction.", "confidence": 0.85, "rule_id": "missing-input-validation-boundary", "source": "ai", "source_engine": "Anthropic PR Review" } ] }, { "label": "after", "analysis_run_id": "public-walkthrough-e62ea22d-6044-4ab2-ab33-19e09ca6238d", "recorded_at": "2026-11-05T19:38:40.385361+00:00", "provider": "anthropic", "model_id": "claude-sonnet-4-6", "model_revision": "unversioned_alias", "prompt_version": "v4", "cache_hit": false, "ai_health": "degraded", "output_validation": { "accepted": 2, "rejected": 1, "rejections": [ { "index": 2, "reason": "end_line is not an added line in the PR diff" } ] }, "coverage": { "complete": false, "files_total": 1, "files_analyzed": 1, "files_omitted": 1, "omissions": [ { "file": "example.py", "reason": "repository_context_unavailable" } ] }, "patch": "--- a/example.py\n+++ b/example.py\n@@ -1,5 +1,8 @@\n def find_users(connection, query_args):\n \"\"\"Look up users from a request query-parameter mapping.\"\"\"\n email = query_args.get(\"email\", \"\")\n- query = f\"SELECT name FROM users WHERE email = '{email}'\"\n- return connection.execute(query).fetchall()\n+ if not isinstance(email, str):\n+ raise ValueError(\"email must be a string\")\n+ return connection.execute(\n+ \"SELECT name FROM users WHERE email = ?\", (email,)\n+ ).fetchall()\n", "accepted_findings": [ { "file": "example.py", "line": 6, "end_line": 8, "severity": "low", "category": "code_quality", "description": "The email value is only checked to be a string instance but is not validated for format, length, or emptiness beyond the default empty string. An empty string or arbitrarily long string will pass through to the database query without error, potentially causing unexpected query results or denial-of-service via oversized inputs.", "suggestion": "Add format and length validation, e.g.: import re; if not email or len(email) > 254 or not re.match(r'^[^@]+@[^@]+\\.[^@]+$', email): raise ValueError('Invalid email address')", "confidence": 0.72, "rule_id": "missing-input-validation", "source": "ai", "source_engine": "Anthropic PR Review" }, { "file": "example.py", "line": 6, "end_line": 8, "severity": "info", "category": "architecture", "description": "Input validation (isinstance check) is co-located with data access logic inside find_users rather than at the boundary where query_args is received. This makes it harder to reuse validation logic and blurs the separation between input handling and data retrieval.", "suggestion": "Move input validation to the caller or a dedicated validation layer, and keep find_users focused solely on database interaction with already-validated parameters.", "confidence": 0.6, "rule_id": "separation-of-concerns", "source": "ai", "source_engine": "Anthropic PR Review" } ] } ], "behavior_validation": { "schema_version": "codestax-sample-validation/v1", "database": "SQLite in-memory", "python_version": "3.11.15", "sqlite_version": "3.46.1", "scope": "These tests establish behavior for this sample and fixture only. They do not prove application-wide security or validate every AI statement.", "tests": [ { "name": "before: ordinary email selects Alice only", "passed": true }, { "name": "before: SQL-like input bypasses filtering", "passed": true }, { "name": "after: ordinary email still selects Alice only", "passed": true }, { "name": "after: SQL-like input remains data", "passed": true }, { "name": "after: apostrophe in legitimate email works", "passed": true }, { "name": "after: missing email has no matching rows in this fixture", "passed": true }, { "name": "before: empty email does not select all rows in this fixture", "passed": true }, { "name": "after: rejects null input", "passed": true }, { "name": "after: rejects list input", "passed": true }, { "name": "after: rejects object input", "passed": true }, { "name": "before: SQLite execute rejects multiple statements and preserves the table", "passed": true } ], "passed": 11, "failed": 0 }, "acquisition_note": { "requested_provider": "openai", "ai_health": "unavailable", "explanation": "An initial isolated OpenAI attempt was unavailable. The paired recorded responses use the already-enabled Anthropic route. No production configuration was changed." }, "reviewer_assessment": [ "The interpolated query permits a filtering bypass in the tested fixture; the bound query prevents that same bypass.", "The model overstates stacked-statement impact for SQLite execute: the tested API rejects multiple statements and the users table remains intact.", "The model suggests an empty email could return the entire table. In this fixture it matches no rows; that general assertion is not supported by this query.", "Authentication, error handling and caller behavior cannot be established from this small diff without repository context.", "The second response has two accepted findings and one rejected location; its AI health is degraded. This is not a clean application-wide security result." ], "limits": "One controlled pair of engine calls, not customer PR evidence, a full delivery test, a repeated-run accuracy benchmark or a release attestation. The model revision is an unversioned alias. Confidence values are model-reported, not calibrated probabilities. SQL-related model claims require human review; see reviewer_assessment and the independent behavior tests." }