New log format

Cyral v2.12.1 introduces a new log structure in order to provide more consistent logging across various data repository types. With the new structure, logs for Snowflake and PostgreSQL, for example, now have a similar structure. This consistency makes it easier to build automation that operates on the log output.


If you have built automations or dashboards that parse Cyral logs (for example, alert forwarders), you may need to rewrite parts of your automations to accept the new format.


Below, we show an example of a policy violation log in which a user tried to read more rows than the policy allowed, resulting in violations for two named LABELs (data locations) covered by the policy. The first example shows the new log format, and the second shows the old format. 


New format

The new format logs activities, each identified by its activityId and activityTime. In the earlier format, the event's timestamp was stored as the timestamp of the request. In the new format, each activity contains a collection of the policyViolations that occurred during that activity.


{
  "activityId": "127.0.0.1:5555:1604365624810254344:43",
  "activityTime": "2020-11-03 01:16:23.919124676 +0000 UTC",
  "activityTimeNanos": 1604366183919124700,
  "activityTypes": [
    "query"
  ],
  "identity": {
    "endUser": "postgres",
    "repoUser": "postgres",
    "dbRole": "postgres"
  },
  "repo": {
    "id": "1iI3T0PV71wMTSQjtG4qAUBR6Ki",
    "name": "pgrepo",
    "type": "postgresql",
    "host": "pgprod.cd6z0yimd7qu.us-west-2.rds.amazonaws.com",
    "port": 5432
  },
  "client": {
    "connectionId": "127.0.0.1:5555:1604365624810254344",
    "connectionTime": "2020-11-03 01:07:04.810254344 +0000 UTC",
    "connectionTimeNanos": 1604365624810254300,
    "host": "127.0.0.1",
    "port": 5555,
    "applicationName": "psql"
  },
  "sidecar": {
    "id": "1iI38aFYOM3xf9MYypT5nQq81Je",
    "name": "default-wrapper",
    "autoScalingGroupInstance": "i-07ae9be1db8863a1b"
  },
  "request": {
    "statement": "SELECT * FROM inpatient_charges",
    "statementType": "SELECT",
    "isSensitive": true,
    "datasetsAccessed": [
      {
        "dataset": "public.inpatient_charges",
        "accessType": "read"
      }
    ],
    "fieldsAccessed": [
      {
        "field": "public.inpatient_charges.provider_state",
        "label": "ST",
        "accessType": "read"
      },
      {
        "field": "public.inpatient_charges.provider_street_address",
        "label": "ADDR",
        "accessType": "read"
      }
    ],
    "searchPath": [
      "postgres",
      "public"
    ]
  },
  "response": {
    "message": "Ok",
    "isError": false,
    "records": 54,
    "bytes": 11214,
    "executionTime": "808.754µs",
    "executionTimeNanos": 808754
  },
  "policyViolated": true,
  "policyViolations": [
    {
      "label": "ST",
      "policyName": "My-Policy",
      "policyId": "1jl63COuljxiUb0kZsIbuM4G6Q5",
      "accessType": "read",
      "selectedIdentity": "user:postgres",
      "reasons": [
        "read is disallowed",
        "54 rows accessed exceeding limit of 0"
      ],
      "severity": "high"
    },
    {
      "label": "ADDR",
      "policyName": "My-Policy",
      "policyId": "1jl63COuljxiUb0kZsIbuM4G6Q5",
      "accessType": "read",
      "selectedIdentity": "user:postgres",
      "reasons": [
        "54 rows accessed exceeding limit of 5"
      ],
      "severity": "high"
    }
  ]
}



Old format

The old format did not use the concept of an activity and instead began with a list of rulesViolated, followed by details about the query that triggered the event.


{
  "rulesViolated":[
    {
      "label":"ST",
      "policyName":"My-Policy",
      "policyId":"1jl63COuljxiUb0kZsIbuM4G6Q5",
      "accessType":"read",
      "selectedIdentity":"user:postgres",
      "reason":[
        "read is disallowed",
        "54 rows accessed exceeding limit of 0"
      ],
      "severity":"high"
    },
    {
      "label":"ADDR",
      "policyName":"My-Policy",
      "policyId":"1jl63COuljxiUb0kZsIbuM4G6Q5",
      "accessType":"read",
      "selectedIdentity":"user:postgres",
      "reason":[
        "54 rows accessed exceeding limit of 5"
      ],
      "severity":"high"
    }
  ],
  "policyViolated":true,
  "sensitiveQuery":true,
  "autoScalingGroupInstance":"i-07ae9be1db8863a1b",
  "queryId":"172.18.0.11:5433:1602079720781392959:14",
  "endUser":"postgres",
  "dbUser":"postgres",
  "dbRole":"postgres",
  "repo":{
    "id":"1iI3T0PV71wMTSQjtG4qAUBR6Ki",
    "name":"pgrepo",
    "type":"postgresql",
    "host":"pgprod.cd6z0yimd7qu.us-west-2.rds.amazonaws.com",
    "port":5432
  },
  "client":{
    "host":"172.18.0.11",
    "port":5433,
    "connectionId":"172.18.0.11:5433:1602079720781392959",
    "connectionTime":"2020-10-07 14:08:40.781392959 +0000 UTC"
  },
  "request":{
    "timestamp":"2020-10-07 14:08:42.723985759 +0000 UTC",
    "timestampMillis":1602079722723,
    "searchPath":[
      "'postgres'",
      "'public'"
    ],
    "statement":"SELECT * FROM inpatient_charges",
    "statementType":"SELECT",
    "tablesReferenced":[
      "public.inpatient_charges"
    ],
    "columnsReferenced":{
      "public.inpatient_charges":[
        "provider_state",
        "provider_street_address"
      ]
    },
    "labelsReferenced":[
      "ST",
      "ADDR"
    ]
  },
  "response":{
    "status":"Ok",
    "rowsAffected":54,
    "bytesAffected":11214,
    "executionTime":"808.754µs",
    "executionTimeMicros":808754
  },
  "activityType":"query"
}





Published 2020-12-15

Login or Signup to post a comment