diff --git a/src/dc-access-logging-app/.devcontainer.json b/src/dc-access-logging-app/.devcontainer.json new file mode 100644 index 00000000..9464a536 --- /dev/null +++ b/src/dc-access-logging-app/.devcontainer.json @@ -0,0 +1,8 @@ +{ + "name": "Data Collection Access Logging", + "dockerComposeFile": "docker-compose.yaml", + "service": "app", + "shutdownAction": "none", + "workspaceFolder": "/workspace", + "remoteUser": "root" +} diff --git a/src/dc-access-logging-app/.gitignore b/src/dc-access-logging-app/.gitignore new file mode 100644 index 00000000..50e608ad --- /dev/null +++ b/src/dc-access-logging-app/.gitignore @@ -0,0 +1,5 @@ +.venv/ +__pycache__/ +*.pyc +*.pyo +.env diff --git a/src/dc-access-logging-app/README.md b/src/dc-access-logging-app/README.md new file mode 100644 index 00000000..f1de75f1 --- /dev/null +++ b/src/dc-access-logging-app/README.md @@ -0,0 +1,89 @@ +# Data Collection Access Logging + +A Flask web app for auditing data collection access grants in Verily Workbench. It queries BigQuery monitoring tables to show who has access, through which groups, and when access was granted. + +## Tabs + +### Forensic: Data Collection (landing page) + +Look up a data collection by ID. Shows the full history of access grants, including: + +- **GROUP** rows: a group was granted access to the data collection. Click the **expand arrow** (▶) to see all members who were already in the group at the time of the grant. +- **MEMBER** rows: an individual who was added to a group after the group was granted access, or removed from a group after the group was granted access. +- **INDIVIDUAL** rows: a user granted access directly. +- **DC Role**: displays the role conferred at the time of the grant (e.g. READER, WRITER, DISCOVERER, OWNER, APPLICATION). If access has since been revoked, the revocation timestamp is shown in red beneath the role. + +Use the **Group Name / Internal Name** toggle above the table to switch between showing the user-facing group name and the internal group identifier. + +Use the **Hide revoked** checkbox to exclude rows where access has been revoked. + +### Group Membership Audit + +Look up a group by its user-facing name. Shows the full history of membership changes — all grants and revocations from the activity log, including timestamps, who acted, and the reason. + +Use the **Current members only** checkbox to show only users who currently hold membership (hides all rows for users whose latest action is a revocation). + +## Org Override + +All tabs include an optional **Org Override** field. By default the app uses the org configured in `config.yaml` or the `DC_ACCESS_ORG_UFID` environment variable (shown in the field's hint text). Enter a different org ID to query that org's tables instead — useful for looking up data collections across orgs without restarting the app. + +## Search Persistence + +When you search for a data collection on the Forensic tab, switching to another tab automatically carries over the search value and org override so you don't have to re-type them. + +## Table Features + +All result tables support: + +- **Keyword filter**: enter one or more space-separated keywords in the filter box. Toggle between **AND** (all keywords must match) and **OR** (any keyword matches) using the segmented toggle next to the filter box. +- **Column filters**: click an underlined column header to filter results by specific values. Use the sort arrow to reorder rows. +- **Sort**: click the sort arrow on any column header to sort ascending/descending. +- **Resize**: drag the right edge of any column header to adjust width. + +## Configuration + +The app reads configuration in this order (first match wins): + +| Setting | Env Var | `config.yaml` key | Fallback | +|---|---|---|---| +| Environment | `DC_ACCESS_ENV` | `env` | `prod` | +| BigQuery data project | `DC_ACCESS_BQ_PROJECT` | `bq_project` | *(none)* | +| BigQuery job project | `DC_ACCESS_JOB_PROJECT` | `job_project` | `wb workspace describe` | +| Organization | `DC_ACCESS_ORG_UFID` | `org` | `wb workspace describe` | + +- **`bq_project`** — the project that hosts the monitoring tables (e.g. `workbench-bq-log-sink`). Used in SQL table references like `` `workbench-bq-log-sink.workbench_monitoring_org_logs_prod.…` ``. +- **`job_project`** — the project where BigQuery jobs are executed and billed. On Workbench, this is automatically resolved to the workspace's Google project. You typically don't have `bigquery.jobs.create` permission on the data project, so jobs must run in your own project. + +Example `config.yaml`: + +```yaml +env: "prod" +bq_project: "workbench-bq-log-sink" +org: "demo" +``` + +The **org** value determines the table suffix. When set (e.g. `demo`), the app queries org-specific tables like `data_collection_access_grants_demo`. When empty, tables have no suffix. + +## Running Locally + +```bash +cd app +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +flask run --host 0.0.0.0 --port 5000 +``` + +Open http://localhost:5000. You need valid GCP credentials with access to the `workbench-bq-log-sink` BigQuery project. + +## Running on Workbench + +The app runs as a devcontainer via `docker-compose.yaml`. Caddy serves as a reverse proxy on port 8080, forwarding to the Flask app on port 5000. + +The org is automatically resolved from `wb workspace describe` if not set via env var or config. + +Environment variables can be set in the `docker-compose.yaml` or passed at launch: + +```bash +DC_ACCESS_ENV=prod DC_ACCESS_ORG_UFID=demo docker-compose up +``` diff --git a/src/dc-access-logging-app/app/.gitignore b/src/dc-access-logging-app/app/.gitignore new file mode 100644 index 00000000..50e608ad --- /dev/null +++ b/src/dc-access-logging-app/app/.gitignore @@ -0,0 +1,5 @@ +.venv/ +__pycache__/ +*.pyc +*.pyo +.env diff --git a/src/dc-access-logging-app/app/Dockerfile b/src/dc-access-logging-app/app/Dockerfile new file mode 100644 index 00000000..5ecdc3f1 --- /dev/null +++ b/src/dc-access-logging-app/app/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 5000 + +CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--workers", "2", "--threads", "4", "app:app"] diff --git a/src/dc-access-logging-app/app/app.py b/src/dc-access-logging-app/app/app.py new file mode 100644 index 00000000..a0f744b9 --- /dev/null +++ b/src/dc-access-logging-app/app/app.py @@ -0,0 +1,1843 @@ +import json +import os +import subprocess +from pathlib import Path + +import yaml +from dotenv import load_dotenv +load_dotenv() + +from flask import Flask, jsonify, render_template_string, request +from flask_cors import CORS +from google.cloud import bigquery + + +def _parse_wb_workspace(): + result = {} + try: + out = subprocess.run( + ["wb", "workspace", "describe", "--format=json"], + capture_output=True, text=True, timeout=10, + ) + if out.returncode == 0 and out.stdout.strip(): + data = json.loads(out.stdout) + if data.get("orgId"): + result["Organization"] = data["orgId"] + if data.get("googleProjectId"): + result["Google project"] = data["googleProjectId"] + except Exception: + pass + return result + +_wb_info = _parse_wb_workspace() + +app = Flask(__name__) +app.config['STRICT_SLASHES'] = False +CORS(app) + +_config_path = Path(__file__).parent / "config.yaml" +_config = {} +if _config_path.exists(): + with open(_config_path) as f: + _config = yaml.safe_load(f) or {} + +ENV = os.environ.get("DC_ACCESS_ENV") or _config.get("env", "prod") +BQ_PROJECT = os.environ.get("DC_ACCESS_BQ_PROJECT") or _config.get("bq_project", "") +JOB_PROJECT = os.environ.get("DC_ACCESS_JOB_PROJECT") or _config.get("job_project", "") or _wb_info.get("Google project", "") +ORG = os.environ.get("DC_ACCESS_ORG_UFID", "").strip() or _config.get("org", "") or _wb_info.get("Organization", "") +ORG_SUFFIX = f"_{ORG}" if ORG else "" + +bq_client = bigquery.Client(project=JOB_PROJECT) if JOB_PROJECT else bigquery.Client() + + + +DC_EXISTS_QUERY = """ +SELECT COUNT(*) AS cnt +FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.data_collection_access_grants{org_suffix}` +WHERE data_collection_user_facing_id = @dc_user_facing_id +""" + + +# if ENV == "dev": +USER_ACTIVITY_LOG_CTES = ''' + user_activity_log_data AS ( + SELECT * + FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.um_user_activity_in_org_log{org_suffix}` + ), +''' +WB_GROUPS_CTE = ''' + wb_groups AS ( + SELECT internal_name, group_name + FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.um_workbench_groups{org_suffix}` + ), +''' + +DC_ACCESS_QUERY = ''' +WITH +{user_activity_log_ctes} + +dc_grants AS ( + SELECT + user_email, + data_collection_name, + data_collection_user_facing_id, + role, + grant_type, + group_email, + group_name, + member_group_name, + user_facing_group_name, + org_id + FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.data_collection_access_grants{org_suffix}` + WHERE data_collection_user_facing_id = @dc_user_facing_id + AND grant_type = 'GROUP' +), + +grant_events AS ( + SELECT + ual.subject_id AS group_internal_name, + ual.change_timestamp, + ual.actor_email AS granted_by, + JSON_VALUE(ual.properties, '$.comment') AS grant_reason, + ( + SELECT REGEXP_EXTRACT(JSON_VALUE(elem, '$.resourceId'), r'(?i)^(.+):USER$') + FROM UNNEST(JSON_EXTRACT_ARRAY(ual.related_resources)) AS elem + WHERE JSON_VALUE(elem, '$.resourceType') = 'PRINCIPAL' + LIMIT 1 + ) AS granted_user_email + FROM user_activity_log_data ual + WHERE ual.change_type = 'GRANT_ROLE_GROUP' +), + +latest_grants AS ( + SELECT + group_internal_name, + granted_user_email, + change_timestamp, + granted_by, + grant_reason, + ROW_NUMBER() OVER ( + PARTITION BY group_internal_name, granted_user_email + ORDER BY change_timestamp DESC + ) AS rn + FROM grant_events + WHERE granted_user_email IS NOT NULL +) + +SELECT +g.user_email, +g.role AS role_on_collection, +g.user_facing_group_name AS group_name, +g.group_email, +g.data_collection_name, +g.data_collection_user_facing_id, +lg.change_timestamp AS access_granted_at, +lg.granted_by, +lg.grant_reason +FROM dc_grants g +LEFT JOIN latest_grants lg +ON lg.group_internal_name = g.member_group_name +AND lg.granted_user_email = g.user_email +AND lg.rn = 1 +ORDER BY +g.role, +g.user_email +''' + + + +GROUP_MEMBERSHIP_QUERY = ''' +WITH +{user_activity_log_ctes} +resolved_group AS ( + SELECT DISTINCT member_group_name + FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.data_collection_access_grants{org_suffix}` + WHERE user_facing_group_name = @group_name +), + +group_events AS ( + SELECT + ual.subject_id AS group_internal_name, + ual.change_type, + ual.change_timestamp, + ual.actor_email AS acted_by, + JSON_VALUE(ual.properties, '$.role') AS group_role, + JSON_VALUE(ual.properties, '$.comment') AS reason, + ( + SELECT REGEXP_EXTRACT(JSON_VALUE(elem, '$.resourceId'), r'(?i)^(.+):USER$') + FROM UNNEST(JSON_EXTRACT_ARRAY(ual.related_resources)) AS elem + WHERE JSON_VALUE(elem, '$.resourceType') = 'PRINCIPAL' + LIMIT 1 + ) AS member_email + FROM user_activity_log_data ual + WHERE ual.change_type IN ('GRANT_ROLE_GROUP', 'REVOKE_ROLE_GROUP') + AND ual.subject_id IN (SELECT member_group_name FROM resolved_group) +), + +ranked_events AS ( + SELECT + *, + ROW_NUMBER() OVER ( + PARTITION BY group_internal_name, member_email + ORDER BY change_timestamp DESC + ) AS rn + FROM group_events + WHERE member_email IS NOT NULL +), + +current_members AS ( + SELECT * + FROM ranked_events + WHERE rn = 1 + AND change_type = 'GRANT_ROLE_GROUP' +) + +SELECT +member_email, +group_internal_name, +group_role, +change_timestamp AS access_granted_at, +acted_by AS granted_by, +reason AS grant_reason +FROM current_members +ORDER BY +change_timestamp DESC +''' + + +GROUP_AUDIT_QUERY = ''' +WITH +{user_activity_log_ctes} + +{wb_groups_cte} + +group_events AS ( + SELECT + ual.change_timestamp, + ual.change_type, + ual.subject_id AS group_internal_name, + wb.group_name AS user_facing_group_name, + ual.actor_email, + UPPER(JSON_VALUE(ual.properties, '$.role')) AS group_role, + JSON_VALUE(ual.properties, '$.comment') AS reason, + ( + SELECT REGEXP_EXTRACT(JSON_VALUE(elem, '$.resourceId'), r'(?i)^(.+):USER$') + FROM UNNEST(JSON_EXTRACT_ARRAY(ual.related_resources)) AS elem + WHERE JSON_VALUE(elem, '$.resourceType') = 'PRINCIPAL' + LIMIT 1 + ) AS member_email + FROM user_activity_log_data ual + LEFT JOIN wb_groups wb ON ual.subject_id = wb.internal_name + WHERE ual.change_type IN ('GRANT_ROLE_GROUP', 'REVOKE_ROLE_GROUP') + AND wb.group_name = @group_name +) + +SELECT + FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', change_timestamp) AS change_timestamp, + CASE + WHEN change_type = 'GRANT_ROLE_GROUP' THEN 'GRANTED' + WHEN change_type = 'REVOKE_ROLE_GROUP' THEN 'REVOKED' + END AS action, + member_email, + user_facing_group_name, + group_internal_name, + group_role, + actor_email AS acted_by, + reason +FROM group_events +WHERE member_email IS NOT NULL +ORDER BY change_timestamp DESC +''' + + +FORENSIC_V2_QUERY = ''' +WITH +{user_activity_log_ctes} +{wb_groups_cte} + +workspace_events AS ( + SELECT + w.change_date AS event_timestamp, + w.change_type, + w.change_subject_id AS subject, + w.actor_email, + UPPER(JSON_VALUE(w.properties, '$.role')) AS role, + CAST(NULL AS STRING) AS reason, + CASE WHEN wb.internal_name IS NOT NULL THEN 'GROUP' ELSE 'INDIVIDUAL' END AS subject_type, + wb.group_name AS group_name, + CASE + WHEN w.change_type = 'GRANT_WORKSPACE_ROLE' THEN 'DC ACCESS GRANTED' + WHEN w.change_type = 'REMOVE_WORKSPACE_ROLE' THEN 'DC ACCESS REVOKED' + END AS action + FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.wsm_workspace_activity_logs{org_suffix}` w + LEFT JOIN wb_groups wb ON REGEXP_EXTRACT(w.change_subject_id, r'^(.+)@verily-bvdp\\.com$') = wb.internal_name + WHERE w.change_type IN ('GRANT_WORKSPACE_ROLE', 'REMOVE_WORKSPACE_ROLE') + AND w.workspace_user_facing_id = @workspace_name +), + +dc_groups AS ( + SELECT DISTINCT REGEXP_EXTRACT(subject, r'^(.+)@verily-bvdp\\.com$') AS internal_name + FROM workspace_events + WHERE subject_type = 'GROUP' +), + +member_events AS ( + SELECT + ual.change_timestamp AS event_timestamp, + ual.change_type, + ( + SELECT REGEXP_EXTRACT(JSON_VALUE(elem, '$.resourceId'), r'(?i)^(.+):USER$') + FROM UNNEST(JSON_EXTRACT_ARRAY(ual.related_resources)) AS elem + WHERE JSON_VALUE(elem, '$.resourceType') = 'PRINCIPAL' + LIMIT 1 + ) AS subject, + ual.actor_email, + UPPER(JSON_VALUE(ual.properties, '$.role')) AS role, + JSON_VALUE(ual.properties, '$.comment') AS reason, + 'MEMBER' AS subject_type, + wb.group_name AS group_name, + CASE + WHEN ual.change_type = 'GRANT_ROLE_GROUP' THEN 'ADDED TO GROUP' + WHEN ual.change_type = 'REVOKE_ROLE_GROUP' THEN 'REMOVED FROM GROUP' + END AS action + FROM user_activity_log_data ual + INNER JOIN dc_groups dg ON ual.subject_id = dg.internal_name + LEFT JOIN wb_groups wb ON ual.subject_id = wb.internal_name + WHERE ual.change_type IN ('GRANT_ROLE_GROUP', 'REVOKE_ROLE_GROUP') +) + +SELECT * FROM ( + SELECT event_timestamp, FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', event_timestamp) AS event_timestamp_fmt, action, subject_type, subject, group_name, role, actor_email, reason + FROM workspace_events + UNION ALL + SELECT event_timestamp, FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', event_timestamp) AS event_timestamp_fmt, action, subject_type, subject, group_name, role, actor_email, reason + FROM member_events + WHERE subject IS NOT NULL +) +ORDER BY event_timestamp DESC +''' + + +WORKSPACE_FORENSIC_QUERY = ''' +WITH +{user_activity_log_ctes} +{wb_groups_cte} + +workspace_grants AS ( + SELECT + w.change_date, + w.change_subject_id, + w.workspace_user_facing_id, + w.org_user_facing_id, + w.actor_email, + CASE + WHEN ( + SELECT MIN(rv.change_date) FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.wsm_workspace_activity_logs{org_suffix}` rv + WHERE rv.change_type = 'REMOVE_WORKSPACE_ROLE' + AND rv.workspace_user_facing_id = w.workspace_user_facing_id + AND rv.change_subject_id = w.change_subject_id + AND UPPER(JSON_VALUE(rv.properties, '$.role')) = UPPER(JSON_VALUE(w.properties, '$.role')) + AND rv.change_date >= w.change_date + ) IS NOT NULL THEN UPPER(JSON_VALUE(w.properties, '$.role')) || ' (REVOKED ' || FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', ( + SELECT MIN(rv.change_date) FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.wsm_workspace_activity_logs{org_suffix}` rv + WHERE rv.change_type = 'REMOVE_WORKSPACE_ROLE' + AND rv.workspace_user_facing_id = w.workspace_user_facing_id + AND rv.change_subject_id = w.change_subject_id + AND UPPER(JSON_VALUE(rv.properties, '$.role')) = UPPER(JSON_VALUE(w.properties, '$.role')) + AND rv.change_date >= w.change_date + )) || ')' + ELSE UPPER(JSON_VALUE(w.properties, '$.role')) + END AS granted_role, + CASE WHEN wb.internal_name IS NOT NULL THEN 'GROUP' ELSE 'DIRECT' END AS dc_grant_type + FROM `workbench-bq-log-sink.workbench_monitoring_org_logs_{env}.wsm_workspace_activity_logs{org_suffix}` w + LEFT JOIN wb_groups wb ON REGEXP_EXTRACT(w.change_subject_id, r'^(.+)@verily-bvdp\.com$') = wb.internal_name + WHERE w.change_type = 'GRANT_WORKSPACE_ROLE' + AND w.workspace_user_facing_id = @workspace_name +), + +group_grants AS ( + SELECT + wg.change_date, + wg.change_subject_id, + wg.workspace_user_facing_id, + wg.org_user_facing_id, + wg.actor_email, + wg.granted_role, + wb.internal_name AS resolved_group_name, + wb.group_name AS user_facing_group_name + FROM workspace_grants wg + JOIN wb_groups wb + ON REGEXP_EXTRACT(wg.change_subject_id, r'^(.+)@verily-bvdp\.com$') = wb.internal_name + WHERE wg.dc_grant_type = 'GROUP' +), + +unresolved_group_grants AS ( + SELECT + wg.change_date, + wg.change_subject_id, + wg.workspace_user_facing_id, + wg.org_user_facing_id, + wg.actor_email, + wg.granted_role + FROM workspace_grants wg + WHERE wg.dc_grant_type = 'GROUP' + AND REGEXP_EXTRACT(wg.change_subject_id, r'^(.+)@verily-bvdp\.com$') NOT IN (SELECT DISTINCT internal_name FROM wb_groups) +), + +member_events AS ( + SELECT + ual.subject_id AS group_internal_name, + ual.change_type, + ual.change_timestamp, + ual.actor_email AS member_granted_by, + UPPER(JSON_VALUE(ual.properties, '$.role')) AS member_role, + ( + SELECT REGEXP_EXTRACT(JSON_VALUE(elem, '$.resourceId'), r'(?i)^(.+):USER$') + FROM UNNEST(JSON_EXTRACT_ARRAY(ual.related_resources)) AS elem + WHERE JSON_VALUE(elem, '$.resourceType') = 'PRINCIPAL' + LIMIT 1 + ) AS member_email + FROM user_activity_log_data ual + WHERE ual.change_type IN ('GRANT_ROLE_GROUP', 'REVOKE_ROLE_GROUP') + AND ual.subject_id IN (SELECT DISTINCT resolved_group_name FROM group_grants) +), + +ranked_members AS ( + SELECT + *, + ROW_NUMBER() OVER ( + PARTITION BY group_internal_name, member_email + ORDER BY change_timestamp DESC + ) AS rn + FROM member_events + WHERE member_email IS NOT NULL +), + +current_members AS ( + SELECT * + FROM ranked_members + WHERE rn = 1 + AND change_type = 'GRANT_ROLE_GROUP' +), + +group_rows AS ( + SELECT + gg.change_date, + 'GROUP' AS grant_level, + gg.change_subject_id AS granted_to, + gg.user_facing_group_name, + gg.resolved_group_name AS internal_name, + gg.workspace_user_facing_id AS workspace, + gg.org_user_facing_id AS org, + gg.actor_email AS granted_by, + gg.change_date AS effective_access_date, + gg.granted_role, + CAST(NULL AS STRING) AS group_member_role + FROM group_grants gg + + UNION ALL + + SELECT + ug.change_date, + 'GROUP' AS grant_level, + ug.change_subject_id AS granted_to, + CAST(NULL AS STRING) AS user_facing_group_name, + REGEXP_EXTRACT(ug.change_subject_id, r'^(.+)@verily-bvdp\.com$') AS internal_name, + ug.workspace_user_facing_id AS workspace, + ug.org_user_facing_id AS org, + ug.actor_email AS granted_by, + ug.change_date AS effective_access_date, + ug.granted_role, + CAST(NULL AS STRING) AS group_member_role + FROM unresolved_group_grants ug +), + +member_rows AS ( + SELECT + cm.change_timestamp AS change_date, + 'MEMBER' AS grant_level, + cm.member_email AS granted_to, + gg.user_facing_group_name, + cm.group_internal_name AS internal_name, + gg.workspace_user_facing_id AS workspace, + gg.org_user_facing_id AS org, + cm.member_granted_by AS granted_by, + CASE + WHEN cm.change_timestamp > gg.group_grant_date THEN cm.change_timestamp + ELSE gg.group_grant_date + END AS effective_access_date, + gg.granted_role, + cm.member_role AS group_member_role + FROM current_members cm + INNER JOIN (SELECT DISTINCT resolved_group_name, user_facing_group_name, workspace_user_facing_id, org_user_facing_id, change_date AS group_grant_date, granted_role FROM group_grants) gg + ON cm.group_internal_name = gg.resolved_group_name + WHERE cm.change_timestamp > gg.group_grant_date +), + +member_events_with_prev AS ( + SELECT + me.*, + LAG(me.change_type) OVER ( + PARTITION BY me.group_internal_name, me.member_email + ORDER BY me.change_timestamp + ) AS prev_change_type + FROM member_events me + WHERE me.member_email IS NOT NULL +), + +revoked_member_rows AS ( + SELECT + mp.change_timestamp AS change_date, + 'MEMBER' AS grant_level, + mp.member_email AS granted_to, + gg.user_facing_group_name, + mp.group_internal_name AS internal_name, + gg.workspace_user_facing_id AS workspace, + gg.org_user_facing_id AS org, + mp.member_granted_by AS granted_by, + mp.change_timestamp AS effective_access_date, + REGEXP_REPLACE(gg.granted_role, r' \(REVOKED.*', '') || ' (REVOKED ' || FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', mp.change_timestamp) || ')' AS granted_role, + mp.member_role AS group_member_role + FROM member_events_with_prev mp + INNER JOIN (SELECT DISTINCT resolved_group_name, user_facing_group_name, workspace_user_facing_id, org_user_facing_id, change_date AS group_grant_date, granted_role FROM group_grants) gg + ON mp.group_internal_name = gg.resolved_group_name + WHERE mp.change_type = 'REVOKE_ROLE_GROUP' + AND mp.prev_change_type = 'GRANT_ROLE_GROUP' + AND mp.change_timestamp > gg.group_grant_date +), + +individual_rows AS ( + SELECT + wg.change_date, + 'INDIVIDUAL' AS grant_level, + wg.change_subject_id AS granted_to, + CAST(NULL AS STRING) AS user_facing_group_name, + CAST(NULL AS STRING) AS internal_name, + wg.workspace_user_facing_id AS workspace, + wg.org_user_facing_id AS org, + wg.actor_email AS granted_by, + wg.change_date AS effective_access_date, + wg.granted_role, + CAST(NULL AS STRING) AS group_member_role + FROM workspace_grants wg + WHERE wg.dc_grant_type != 'GROUP' +) + +SELECT FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', change_date) AS change_date, grant_level, granted_to, user_facing_group_name, internal_name, workspace, org, granted_by, effective_access_date, granted_role, group_member_role FROM group_rows +UNION ALL +SELECT FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', change_date) AS change_date, grant_level, granted_to, user_facing_group_name, internal_name, workspace, org, granted_by, effective_access_date, granted_role, group_member_role FROM member_rows +UNION ALL +SELECT FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', change_date) AS change_date, grant_level, granted_to, user_facing_group_name, internal_name, workspace, org, granted_by, effective_access_date, granted_role, group_member_role FROM revoked_member_rows +UNION ALL +SELECT FORMAT_TIMESTAMP('%Y-%m-%d %H:%M:%S', change_date) AS change_date, grant_level, granted_to, user_facing_group_name, internal_name, workspace, org, granted_by, effective_access_date, granted_role, group_member_role FROM individual_rows +ORDER BY change_date DESC NULLS LAST +''' + +GROUP_MEMBERS_AT_QUERY = ''' +WITH +{user_activity_log_ctes} +group_events AS ( + SELECT + ual.subject_id AS group_internal_name, + ual.change_type, + ual.change_timestamp, + ual.actor_email AS acted_by, + JSON_VALUE(ual.properties, '$.comment') AS reason, + ( + SELECT REGEXP_EXTRACT(JSON_VALUE(elem, '$.resourceId'), r'(?i)^(.+):USER$') + FROM UNNEST(JSON_EXTRACT_ARRAY(ual.related_resources)) AS elem + WHERE JSON_VALUE(elem, '$.resourceType') = 'PRINCIPAL' + LIMIT 1 + ) AS member_email + FROM user_activity_log_data ual + WHERE ual.change_type IN ('GRANT_ROLE_GROUP', 'REVOKE_ROLE_GROUP') + AND ual.subject_id = @group_name + AND ual.change_timestamp <= @as_of_timestamp +), + +ranked AS ( + SELECT + *, + ROW_NUMBER() OVER ( + PARTITION BY member_email + ORDER BY change_timestamp DESC + ) AS rn + FROM group_events + WHERE member_email IS NOT NULL +) + +SELECT + member_email, + change_timestamp AS added_at, + acted_by AS added_by, + reason +FROM ranked +WHERE rn = 1 + AND change_type = 'GRANT_ROLE_GROUP' +ORDER BY change_timestamp DESC +''' + + + + +######################################################################################################################################################## + + +BASE_TEMPLATE = """ + + + + + + Data Collection Access Logging + + + + + + + + +
+
+ Forensic: Data Collection + Group Membership Audit +
+ + {% block content %}{% endblock %} +
+ + + + + + +""" + +DC_ACCESS_CONTENT = """ +
+

Data Collection Access by Group Membership

+

Shows which individuals hold which roles on a data collection, + via which group memberships, and when that access was granted.

+
+ +
+
+
+ + +
+
+ + +
+ +
+
+ + {% if error %} +
{{ error }}
+ {% endif %} + + {% if rows is not none and rows|length == 0 %} +
No group-based access found for data collection "{{ dc_id }}".
+ {% endif %} + + {% if rows %} + + + + + + + + + + + + + {% for r in rows %} + + + + + + + + + {% endfor %} + +
User EmailRole on CollectionGroup NameAccess Granted AtGranted ByGrant Reason
{{ r.user_email }}{{ r.role_on_collection }}{{ r.group_name }}{{ r.access_granted_at or '—' }}{{ r.granted_by or '—' }}{{ r.grant_reason or '—' }}
+ {% endif %} +""" + +GROUP_MEMBERSHIP_CONTENT = """ +
+

Group Membership with Grant Timestamps

+

Lists all individuals who currently have access via membership + in a group and when that access was conferred.

+
+ +
+
+
+ + +
+
+ + +
+ +
+
+ + {% if error %} +
{{ error }}
+ {% endif %} + + {% if rows is not none and rows|length == 0 %} +
No current members found for this group.
+ {% endif %} + + {% if rows %} + + + + + + + + + + + + + {% for r in rows %} + + + + + + + + + {% endfor %} + +
Member EmailGroup Internal NameGroup RoleAccess Granted AtGranted ByGrant Reason
{{ r.member_email }}{{ r.group_internal_name }}{{ r.group_role }}{{ r.access_granted_at or '—' }}{{ r.granted_by or '—' }}{{ r.grant_reason or '—' }}
+ {% endif %} + +""" + + +FORENSIC_V2_CONTENT = """ +
+

Forensic View v2: Full Access Timeline

+

Unified timeline of data collection access — workspace-level grants/revocations and group membership changes from the activity log.

+
+ +
+
+
+ + +
+
+ + +
+ +
+
+ + {% if error %} +
{{ error }}
+ {% endif %} + + {% if rows is not none and rows|length == 0 %} +
No events found for data collection "{{ workspace_name }}".
+ {% endif %} + + {% if rows %} +
+
+ Action: + + + + +
+
+ Type: + + + +
+
+ Status: + +
+
+ + + + + + + + + + + + + + + {% for r in rows %} + + + + + + + + + + + {% endfor %} + +
TimestampActionTypeSubjectGroupRoleActed ByReason
{{ r.event_timestamp_fmt or '—' }}{{ r.action }}{{ r.subject_type }}{{ r.subject }}{{ r.group_name or '—' }}{{ r.role or '—' }}{{ r.actor_email or '—' }}{{ r.reason or '—' }}
+ + + {% endif %} +""" + + +GROUP_AUDIT_CONTENT = """ +
+

Group Membership Audit

+

Full history of group membership changes — shows all grants and revocations from the activity log.

+
+ +
+
+
+ + +
+
+ + +
+ +
+
+ + {% if error %} +
{{ error }}
+ {% endif %} + + {% if rows is not none and rows|length == 0 %} +
No membership events found for group "{{ group_name }}".
+ {% endif %} + + {% if rows %} + {% set ga_roles = [] %} + {% for r in rows %}{% if r.group_role and r.group_role not in ga_roles %}{% if ga_roles.append(r.group_role) %}{% endif %}{% endif %}{% endfor %} +
+ +
+ + + + + + + + + + + + + + {% for r in rows %} + + + + + + + + + + {% endfor %} + +
Timestamp +
Action +
+ + +
+
+ 		Member EmailGroup Name +
Role +
+ {% for role in ga_roles|sort %} + + {% endfor %} +
+
+ 		Acted ByReason
{{ r.change_timestamp or '—' }}{{ r.action }}{{ r.member_email }}{{ r.user_facing_group_name or '—' }}{{ r.group_role or '—' }}{{ r.acted_by or '—' }}{{ r.reason or '—' }}
+ + + {% endif %} +""" + + +WORKSPACE_FORENSIC_CONTENT = """ +
+

Forensic View: Data Collection Access Grant History

+

Shows all data collection access grants — groups, their members at time of grant, and individual users. Expand a group row to see who was already in the group when it was granted access.

+
+ +
+
+
+ + +
+
+ + +
+ +
+
+ + {% if error %} +
{{ error }}
+ {% endif %} + + {% if rows is not none and rows|length == 0 %} +
No access grants found for data collection "{{ workspace_name }}".
+ {% endif %} + + {% if rows %} + {% set roles = rows|map(attribute='granted_role')|map('default', '')|list %} + {% set unique_roles = [] %} + {% for r in roles %}{% set base = r.split(' (')[0] %}{% if base and base not in unique_roles %}{% if unique_roles.append(base) %}{% endif %}{% endif %}{% endfor %} +
+
+ Column: + + +
+ +
+ + + + + + + + + + + + + + + + {% for r in rows %} + + + + + + + + + + + + {% endfor %} + +
Event Date +
Grant Level +
+ + + +
+
+ 		Group NameInternal NameGranted To +
DC Role +
+ {% for role in unique_roles|sort %} + + {% endfor %} +
+
+ 		OrgGranted By
{% if r.grant_level == 'GROUP' and r.internal_name %}▶{% endif %}{{ r.change_date or '—' }}{{ r.grant_level }}{{ r.user_facing_group_name or '—' }}{{ r.internal_name or '—' }}{{ r.granted_to }}{% if r.granted_role and '(REVOKED' in r.granted_role %}{{ r.granted_role.split(' (')[0] }}
REVOKED {{ r.granted_role.split('REVOKED ')[1].rstrip(')') }}{% else %}{{ r.granted_role or '—' }}{% endif %}		{{ r.org or '—' }}{{ r.granted_by or '—' }}
+ + + {% endif %} +""" + + +def _render(content_template, **kwargs): + kwargs.setdefault("env", ENV) + full_template = BASE_TEMPLATE.replace("{% block content %}{% endblock %}", content_template) + return render_template_string(full_template, **kwargs) + + +def _run_query(query_template, params, org_suffix=None): + if org_suffix is None: + org_suffix = ORG_SUFFIX + sql = query_template.replace('{user_activity_log_ctes}', USER_ACTIVITY_LOG_CTES) + sql = sql.replace('{wb_groups_cte}', WB_GROUPS_CTE) + sql = sql.format(env=ENV, bq_project=BQ_PROJECT, org_suffix=org_suffix) + job_config = bigquery.QueryJobConfig(query_parameters=params) + result = bq_client.query(sql, job_config=job_config).result() + return [dict(row) for row in result] + + +@app.route("/dc-access") +def dc_access(): + dc_id = request.args.get("dc_id", "").strip() + org_override = request.args.get("org_override", "").strip() + org_suffix = f"_{org_override}" if org_override else None + + if not dc_id: + return _render(DC_ACCESS_CONTENT, active_tab="dc", dc_id=None, rows=None, error=None, + org_override=org_override, default_org=ORG) + + try: + dc_params = [bigquery.ScalarQueryParameter("dc_user_facing_id", "STRING", dc_id)] + exists = _run_query(DC_EXISTS_QUERY, dc_params, org_suffix=org_suffix) + if exists[0]["cnt"] == 0: + return _render(DC_ACCESS_CONTENT, active_tab="dc", dc_id=dc_id, rows=None, + error=f'Data collection "{dc_id}" was not found.', + org_override=org_override, default_org=ORG) + rows = _run_query(DC_ACCESS_QUERY, dc_params, org_suffix=org_suffix) + except Exception as e: + return _render(DC_ACCESS_CONTENT, active_tab="dc", dc_id=dc_id, rows=None, error=str(e), + org_override=org_override, default_org=ORG) + + return _render(DC_ACCESS_CONTENT, active_tab="dc", dc_id=dc_id, rows=rows, error=None, + org_override=org_override, default_org=ORG) + + +@app.route("/group") +def group_membership(): + group_name = request.args.get("group_name", "").strip() + org_override = request.args.get("org_override", "").strip() + org_suffix = f"_{org_override}" if org_override else None + + if not group_name: + return _render(GROUP_MEMBERSHIP_CONTENT, active_tab="group", group_name=None, rows=None, error=None, + org_override=org_override, default_org=ORG) + + try: + rows = _run_query(GROUP_MEMBERSHIP_QUERY, [ + bigquery.ScalarQueryParameter("group_name", "STRING", group_name), + ], org_suffix=org_suffix) + except Exception as e: + return _render(GROUP_MEMBERSHIP_CONTENT, active_tab="group", group_name=group_name, rows=None, error=str(e), + org_override=org_override, default_org=ORG) + + return _render(GROUP_MEMBERSHIP_CONTENT, active_tab="group", group_name=group_name, rows=rows, error=None, + org_override=org_override, default_org=ORG) + + + +@app.route("/forensic-v2") +def forensic_v2_view(): + workspace_name = request.args.get("workspace_name", "").strip() + org_override = request.args.get("org_override", "").strip() + org_suffix = f"_{org_override}" if org_override else None + + if not workspace_name: + return _render(FORENSIC_V2_CONTENT, active_tab="forensic_v2", + workspace_name=None, rows=None, error=None, + org_override=org_override, default_org=ORG) + + try: + rows = _run_query(FORENSIC_V2_QUERY, [ + bigquery.ScalarQueryParameter("workspace_name", "STRING", workspace_name), + ], org_suffix=org_suffix) + except Exception as e: + return _render(FORENSIC_V2_CONTENT, active_tab="forensic_v2", + workspace_name=workspace_name, rows=None, error=str(e), + org_override=org_override, default_org=ORG) + + return _render(FORENSIC_V2_CONTENT, active_tab="forensic_v2", + workspace_name=workspace_name, rows=rows, error=None, + org_override=org_override, default_org=ORG) + + +@app.route("/group-audit") +def group_audit(): + group_name = request.args.get("group_name", "").strip() + org_override = request.args.get("org_override", "").strip() + org_suffix = f"_{org_override}" if org_override else None + + if not group_name: + return _render(GROUP_AUDIT_CONTENT, active_tab="group_audit", group_name=None, rows=None, error=None, + org_override=org_override, default_org=ORG) + + try: + rows = _run_query(GROUP_AUDIT_QUERY, [ + bigquery.ScalarQueryParameter("group_name", "STRING", group_name), + ], org_suffix=org_suffix) + except Exception as e: + return _render(GROUP_AUDIT_CONTENT, active_tab="group_audit", group_name=group_name, rows=None, error=str(e), + org_override=org_override, default_org=ORG) + + return _render(GROUP_AUDIT_CONTENT, active_tab="group_audit", group_name=group_name, rows=rows, error=None, + org_override=org_override, default_org=ORG) + + +@app.route("/") +def workspace_forensic_view(): + workspace_name = request.args.get("workspace_name", "").strip() + org_override = request.args.get("org_override", "").strip() + org_suffix = f"_{org_override}" if org_override else None + + if not workspace_name: + return _render(WORKSPACE_FORENSIC_CONTENT, active_tab="ws_forensic", + workspace_name=None, rows=None, error=None, + org_override=org_override, default_org=ORG) + + try: + rows = _run_query(WORKSPACE_FORENSIC_QUERY, [ + bigquery.ScalarQueryParameter("workspace_name", "STRING", workspace_name), + ], org_suffix=org_suffix) + except Exception as e: + return _render(WORKSPACE_FORENSIC_CONTENT, active_tab="ws_forensic", + workspace_name=workspace_name, rows=None, error=str(e), + org_override=org_override, default_org=ORG) + + return _render(WORKSPACE_FORENSIC_CONTENT, active_tab="ws_forensic", + workspace_name=workspace_name, rows=rows, error=None, + org_override=org_override, default_org=ORG) + + +@app.route("/api/group-members-at") +def group_members_at(): + group_name = request.args.get("group_name", "").strip() + timestamp = request.args.get("timestamp", "").strip() + if not group_name or not timestamp: + return jsonify({"error": "group_name and timestamp are required"}), 400 + + try: + rows = _run_query(GROUP_MEMBERS_AT_QUERY, [ + bigquery.ScalarQueryParameter("group_name", "STRING", group_name), + bigquery.ScalarQueryParameter("as_of_timestamp", "TIMESTAMP", timestamp), + ]) + for row in rows: + for k, v in row.items(): + if hasattr(v, 'isoformat'): + row[k] = v.isoformat() + return jsonify(rows) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + +@app.route("/health") +def health(): + return {"status": "ok"} + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=5000) diff --git a/src/dc-access-logging-app/app/config.yaml b/src/dc-access-logging-app/app/config.yaml new file mode 100644 index 00000000..02fc2681 --- /dev/null +++ b/src/dc-access-logging-app/app/config.yaml @@ -0,0 +1,3 @@ +env: "prod" +org: "demo" +bq_project: "workbench-bq-log-sink" diff --git a/src/dc-access-logging-app/app/requirements.txt b/src/dc-access-logging-app/app/requirements.txt new file mode 100644 index 00000000..549eab4d --- /dev/null +++ b/src/dc-access-logging-app/app/requirements.txt @@ -0,0 +1,6 @@ +flask==3.1.1 +flask-cors==5.0.1 +gunicorn==23.0.0 +google-cloud-bigquery==3.31.0 +python-dotenv==1.1.0 +pyyaml==6.0.2 diff --git a/src/dc-access-logging-app/devcontainer-template.json b/src/dc-access-logging-app/devcontainer-template.json new file mode 100644 index 00000000..9bcc0544 --- /dev/null +++ b/src/dc-access-logging-app/devcontainer-template.json @@ -0,0 +1,6 @@ +{ + "id": "dc-access-logging-app", + "version": "1.0.0", + "name": "Data Collection Access Logging", + "description": "Flask application for viewing data collection access logs from BigQuery" +} diff --git a/src/dc-access-logging-app/docker-compose.yaml b/src/dc-access-logging-app/docker-compose.yaml new file mode 100644 index 00000000..f4a1b41d --- /dev/null +++ b/src/dc-access-logging-app/docker-compose.yaml @@ -0,0 +1,20 @@ +services: + app: + # The container name must be "application-server" + container_name: "application-server" + build: + context: ./app + restart: always + environment: + DC_ACCESS_ENV: "${DC_ACCESS_ENV:-prod}" + DC_ACCESS_ORG_UFID: "${DC_ACCESS_ORG_UFID:-}" + ports: + - 8080:5000 + networks: + - app-network + +networks: + # The Docker network must be named "app-network". This is an external network + # that is created outside of this docker-compose file. + app-network: + external: true