Schema Info

Schema Name: sfdl_stage_segment.reactfoot_polluted_user

created: 2018-03-12 17:01:40
disabled: False
format: raw
id: 432
is_partitioned: True
is_schema_datatyped: False

name:

{
    "database": "sfdl_stage_segment",
    "table": "reactfoot_polluted_user"
}

type: segment

Schema Versions: 3

Version 199040

created: 2019-03-16 00:30:41

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_stage_segment`",
    "DROP TABLE IF EXISTS `sfdl_stage_segment.reactfoot_polluted_user`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_stage_segment.reactfoot_polluted_user`\n(\n`timestamp`STRING,`userId`STRING,`integrations`struct<`_empty_`:STRING>,`receivedAt`STRING,`original_key`STRING,`_origin`STRING,`properties`struct<`errorMessage`:STRING,`message`:STRING,`experiments`:STRING,`experimentList`:array>,`projectId`STRING,`event`STRING,`_metadata`struct<`nodeVersion`:STRING>,`version`STRING,`originalTimestamp`STRING,`context`struct<`library`:struct<`version`:STRING,`name`:STRING>>,`messageId`STRING,`writeKey`STRING,`type`STRING,`channel`STRING,`sentAt`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user'\n",
    "MSCK REPAIR TABLE sfdl_stage_segment.reactfoot_polluted_user"
]

hive_path: s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user
id: 199040
processed: 1

raw_data:

{
    "_metadata": {
        "nodeVersion": "8.6.0"
    },
    "_origin": "2019-03-16T05:30:31Z|s3://sfdl-segment-reactfoot-stage/segment-logs/nQDzdrZI1K/1552694400000/1552714151860.b7a80a25c764.dev.3297d5b3-52a7-49ca-aad2-1e88a427ec50.gz|segment_separator:37",
    "channel": "server",
    "context": {
        "library": {
            "name": "analytics-node",
            "version": "3.2.0"
        }
    },
    "event": "polluted_user",
    "integrations": {},
    "messageId": "node-mV13DtnA2wDjoDY9Z1zGZnJZG6XPHfTL",
    "originalTimestamp": "2019-03-16T05:16:39.951Z",
    "original_key": "sfdl-segment-reactfoot-stage/segment-logs/nQDzdrZI1K/1521417600000/1521478868147.25c3a9590606.1.11.1.353a5e02-2d56-4432-ba58-0be3470f6521.gz",
    "projectId": "nQDzdrZI1K",
    "properties": {
        "errorMessage": "504 - \"upstream request timeout\"",
        "experimentList": [
            "red_716_tracking_prerender",
            "sf_4830_aa_phlag_band_aid",
            "sf_6852_serp_results",
            "sf_7018_upsellit",
            "sf_7046_recommended_facility_badge",
            "sf_7155_two_step_checkout_v3",
            "sf_7249_guided_unit_selection",
            "sf_7371_nvf_100_donuts",
            "sf_7485_scarcity_treatment"
        ],
        "experiments": "red_716_tracking_prerender,sf_4830_aa_phlag_band_aid,sf_6852_serp_results,sf_7018_upsellit,sf_7046_recommended_facility_badge,sf_7155_two_step_checkout_v3,sf_7249_guided_unit_selection,sf_7371_nvf_100_donuts,sf_7485_scarcity_treatment",
        "message": "fetchExperimentVariations failure"
    },
    "receivedAt": "2019-03-16T05:16:49.955Z",
    "sentAt": "2019-03-16T05:16:49.593Z",
    "timestamp": "2019-03-16T05:16:40.313Z",
    "type": "track",
    "userId": "bot",
    "version": 2,
    "writeKey": "gPzkrXYTAk3mq186YXZjfn5Ad1SwqFlN"
}

schema_attributes:

{
    "_origin": "string",
    "channel": "string",
    "context.library.name": "string",
    "context.library.version": "string",
    "event": "string",
    "messageId": "string",
    "originalTimestamp": "string",
    "original_key": "string",
    "projectId": "string",
    "properties.errorMessage": "string",
    "properties.experimentList": [
        "string"
    ],
    "properties.experiments": "string",
    "properties.message": "string",
    "receivedAt": "string",
    "sentAt": "string",
    "timestamp": "string",
    "type": "string",
    "userId": "string",
    "version": "string",
    "writeKey": "string"
}

schema_name_id: 432
schema_scan_id: 1952968
updated: 2019-03-16 00:31:27

Version 2980

created: 2018-03-20 11:04:50

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_stage_segment`",
    "DROP TABLE IF EXISTS `sfdl_stage_segment.reactfoot_polluted_user`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_stage_segment.reactfoot_polluted_user`\n(\n `writeKey`  string , `timestamp`  string , `messageId`  string , `userId`  string , `event`  string , `receivedAt`  string ,`_metadata`   struct< `nodeVersion` : string >, `version`  string , `originalTimestamp`  string ,`context`   struct<`library` :  struct< `version` : string , `name` : string >>, `projectId`  string , `original_key`  string , `_origin`  string , `type`  string ,`properties`   struct< `experiments` : string , `message` : string , `errorMessage` : string , `experimentList`:array >, `channel`  string , `sentAt`  string \n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user'\n",
    "MSCK REPAIR TABLE sfdl_stage_segment.reactfoot_polluted_user"
]

hive_path: s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user
id: 2980
processed: 1

raw_data:

{
    "_metadata": {
        "nodeVersion": "8.6.0"
    },
    "_origin": "2018-03-20T16:03:39Z|s3://sfdl-segment-reactfoot-stage/segment-logs/nQDzdrZI1K/1521504000000/1521561675828.efa9ce6076c4.1.11.1.e5d62d3f-3452-4bcc-b536-f122e54b3745.gz|segment_separator:33",
    "channel": "server",
    "context": {
        "library": {
            "name": "analytics-node",
            "version": "3.2.0"
        }
    },
    "event": "polluted_user",
    "integrations": {},
    "messageId": "node-kT0Rklx91BGBIFNzwayFoy11kmNeEBri",
    "originalTimestamp": "2018-03-20T15:51:11.872Z",
    "original_key": "sfdl-segment-reactfoot-stage/segment-logs/nQDzdrZI1K/1521417600000/1521478868147.25c3a9590606.1.11.1.353a5e02-2d56-4432-ba58-0be3470f6521.gz",
    "projectId": "nQDzdrZI1K",
    "properties": {
        "errorMessage": "Error: ETIMEDOUT",
        "experimentList": [
            "sf_4137_sms_opt_in",
            "sf_4161_post_res_paid",
            "sf_4606_post_res_sms",
            "sf_4642_lock_incentive",
            "sf_4651_stacked_ui_links",
            "sf_4787_facility_page",
            "sf_4788_search_filters",
            "sf_4792_remove_date_from_search",
            "sf_4830_aa_phlag_band_aid"
        ],
        "experiments": "sf_4137_sms_opt_in,sf_4161_post_res_paid,sf_4606_post_res_sms,sf_4642_lock_incentive,sf_4651_stacked_ui_links,sf_4787_facility_page,sf_4788_search_filters,sf_4792_remove_date_from_search,sf_4830_aa_phlag_band_aid",
        "message": "fetchExperimentVariations failure"
    },
    "receivedAt": "2018-03-20T15:51:13.804Z",
    "sentAt": "2018-03-20T15:51:11.873Z",
    "timestamp": "2018-03-20T15:51:13.803Z",
    "type": "track",
    "userId": "mitchhh",
    "version": 2,
    "writeKey": "gPzkrXYTAk3mq186YXZjfn5Ad1SwqFlN"
}

schema_attributes:

{
    "_origin": "string",
    "channel": "string",
    "context.library.name": "string",
    "context.library.version": "string",
    "event": "string",
    "messageId": "string",
    "originalTimestamp": "string",
    "original_key": "string",
    "projectId": "string",
    "properties.errorMessage": "string",
    "properties.experimentList": "",
    "properties.experiments": "string",
    "properties.message": "string",
    "receivedAt": "string",
    "sentAt": "string",
    "timestamp": "string",
    "type": "string",
    "userId": "string",
    "version": "string",
    "writeKey": "string"
}

schema_name_id: 432
schema_scan_id: None
updated: 2018-05-24 16:44:04

Version 2847

created: 2018-03-19 12:01:56

ddl:

[
    "CREATE DATABASE IF NOT EXISTS `sfdl_stage_segment`",
    "DROP TABLE IF EXISTS `sfdl_stage_segment.reactfoot_polluted_user`",
    "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_stage_segment.reactfoot_polluted_user`\n(\n `timestamp`  string , `messageId`  string , `userId`  string ,`properties`   struct< `experiments` : string , `message` : string , `errorMessage` : string , `experimentList`:array >, `receivedAt`  string ,`_metadata`   struct< `nodeVersion` : string >, `version`  string , `originalTimestamp`  string ,`context`   struct<`library` :  struct< `version` : string , `name` : string >>, `projectId`  string , `original_key`  string , `writeKey`  string , `type`  string , `event`  string , `channel`  string , `sentAt`  string \n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user'\n",
    "MSCK REPAIR TABLE sfdl_stage_segment.reactfoot_polluted_user"
]

hive_path: s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user
id: 2847
processed: 1

raw_data:

{
    "_metadata": {
        "nodeVersion": "8.6.0"
    },
    "channel": "server",
    "context": {
        "library": {
            "name": "analytics-node",
            "version": "3.2.0"
        }
    },
    "event": "polluted_user",
    "integrations": {},
    "messageId": "node-2FyBEnIqaHEympNvIS6PGWIPNC3TXPIx",
    "originalTimestamp": "2018-03-19T16:22:56.878Z",
    "original_key": "sfdl-segment-reactfoot-stage/segment-logs/nQDzdrZI1K/1521417600000/1521478868147.25c3a9590606.1.11.1.353a5e02-2d56-4432-ba58-0be3470f6521.gz",
    "projectId": "nQDzdrZI1K",
    "properties": {
        "errorMessage": "Error: ETIMEDOUT",
        "experimentList": [
            "sf_4137_sms_opt_in",
            "sf_4161_post_res_paid",
            "sf_4606_post_res_sms",
            "sf_4642_lock_incentive",
            "sf_4651_stacked_ui_links",
            "sf_4787_facility_page",
            "sf_4788_search_filters",
            "sf_4792_remove_date_from_search",
            "sf_4830_aa_phlag_band_aid"
        ],
        "experiments": "sf_4137_sms_opt_in,sf_4161_post_res_paid,sf_4606_post_res_sms,sf_4642_lock_incentive,sf_4651_stacked_ui_links,sf_4787_facility_page,sf_4788_search_filters,sf_4792_remove_date_from_search,sf_4830_aa_phlag_band_aid",
        "message": "fetchExperimentVariations failure"
    },
    "receivedAt": "2018-03-19T16:22:57.204Z",
    "sentAt": "2018-03-19T16:22:56.879Z",
    "timestamp": "2018-03-19T16:22:57.203Z",
    "type": "track",
    "userId": "null",
    "version": 2,
    "writeKey": "gPzkrXYTAk3mq186YXZjfn5Ad1SwqFlN"
}

schema_attributes:

{
    "channel": "string",
    "context.library.name": "string",
    "context.library.version": "string",
    "event": "string",
    "messageId": "string",
    "originalTimestamp": "string",
    "original_key": "string",
    "projectId": "string",
    "properties.errorMessage": "string",
    "properties.experimentList": "",
    "properties.experiments": "string",
    "properties.message": "string",
    "receivedAt": "string",
    "sentAt": "string",
    "timestamp": "string",
    "type": "string",
    "userId": "string",
    "version": "string",
    "writeKey": "string"
}

schema_name_id: 432
schema_scan_id: None
updated: 2018-03-19 12:02:07

Schema Scans: 1

Last at 2019-03-16 00:30:40

Scan 1952968

duration: 0:00:27.387867

exit_message:

{
    "ddl_changed": true,
    "ingested_partitions": 94,
    "partitions_applied": false,
    "success": true
}

id: 1952968

payload:

{
    "datatype_dict": null,
    "file_format": "json",
    "partition": {
        "location": "s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2019-03-16/",
        "name": "dt",
        "type": "string",
        "value": "2019-03-16"
    },
    "s3": {
        "bucket": "sfdl-segment-reactfoot-stage",
        "hive_path": "s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user",
        "key": "separated_dt/polluted_user/dt=2019-03-16/1552714151860.b7a80a25c764.dev.3297d5b3-52a7-49ca-aad2-1e88a427ec50.json.gz"
    },
    "schema": {
        "name": "sfdl_stage_segment.reactfoot_polluted_user",
        "type": "segment"
    }
}

running: False
schema_name_id: 432
start_time: 2019-03-16 00:30:40
success: True
trace_id: 15016622253798341730

Partitions:

count: 3

ddl:

[
    "ALTER TABLE\n    `sfdl_stage_segment.reactfoot_polluted_user`\nADD\n\n\n PARTITION (dt='2024-12-13') location 's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-13/'\n PARTITION (dt='2024-12-23') location 's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-23/'\n PARTITION (dt='2024-12-24') location 's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-24/'\n;"
]

name: dt

state:

[
    {
        "info": {
            "id": 586617,
            "location": "s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-13/",
            "name": "dt",
            "value": "2024-12-13"
        },
        "state": {
            "arrival": {
                "athena_discovery": true,
                "first_data": "2024-12-13 15:38:51",
                "last_data": "2024-12-13 15:47:08"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": true
            }
        }
    },
    {
        "info": {
            "id": 588645,
            "location": "s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-23/",
            "name": "dt",
            "value": "2024-12-23"
        },
        "state": {
            "arrival": {
                "athena_discovery": true,
                "first_data": "2024-12-23 10:54:01",
                "last_data": "2024-12-23 12:16:19"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": true
            }
        }
    },
    {
        "info": {
            "id": 588813,
            "location": "s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-24/",
            "name": "dt",
            "value": "2024-12-24"
        },
        "state": {
            "arrival": {
                "athena_discovery": true,
                "first_data": "2024-12-24 10:28:49",
                "last_data": "2024-12-24 10:39:16"
            },
            "parquet": {
                "first_performed": null,
                "in_progress": null,
                "job_id": null,
                "last_performed": null,
                "queued": true
            }
        }
    }
]

type: string

Schema:

attributes:

{
    "_origin": "string",
    "channel": "string",
    "context.library.name": "string",
    "context.library.version": "string",
    "event": "string",
    "messageId": "string",
    "originalTimestamp": "string",
    "original_key": "string",
    "projectId": "string",
    "properties.errorMessage": "string",
    "properties.experimentList": [
        "string"
    ],
    "properties.experiments": "string",
    "properties.message": "string",
    "receivedAt": "string",
    "sentAt": "string",
    "timestamp": "string",
    "type": "string",
    "userId": "string",
    "version": "string",
    "writeKey": "string"
}

created: 2019-03-16 00:30:41

ddl:

{
    "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_stage_segment`",
    "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_stage_segment.reactfoot_polluted_user`\n(\n`timestamp`STRING,`userId`STRING,`integrations`struct<`_empty_`:STRING>,`receivedAt`STRING,`original_key`STRING,`_origin`STRING,`properties`struct<`errorMessage`:STRING,`message`:STRING,`experiments`:STRING,`experimentList`:array>,`projectId`STRING,`event`STRING,`_metadata`struct<`nodeVersion`:STRING>,`version`STRING,`originalTimestamp`STRING,`context`struct<`library`:struct<`version`:STRING,`name`:STRING>>,`messageId`STRING,`writeKey`STRING,`type`STRING,`channel`STRING,`sentAt`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user'\n",
    "drop_table": "DROP TABLE IF EXISTS `sfdl_stage_segment.reactfoot_polluted_user`",
    "repair_table": "MSCK REPAIR TABLE sfdl_stage_segment.reactfoot_polluted_user"
}

Server:

hostname: schemamanager-5c55c886bf-h2ts4
name: schema_manager
process_start: 2025-10-21 13:03:27
sf_env: stage
version: 2112174301

Raw:

{
    "name": {
        "created": "2018-03-12 17:01:40",
        "disabled": false,
        "format": "raw",
        "id": 432,
        "is_partitioned": true,
        "is_schema_datatyped": false,
        "name": {
            "database": "sfdl_stage_segment",
            "table": "reactfoot_polluted_user"
        },
        "type": "segment"
    },
    "partition": {
        "count": 3,
        "ddl": [
            "ALTER TABLE\n    `sfdl_stage_segment.reactfoot_polluted_user`\nADD\n\n\n PARTITION (dt='2024-12-13') location 's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-13/'\n PARTITION (dt='2024-12-23') location 's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-23/'\n PARTITION (dt='2024-12-24') location 's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-24/'\n;"
        ],
        "name": "dt",
        "state": [
            {
                "info": {
                    "id": 586617,
                    "location": "s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-13/",
                    "name": "dt",
                    "value": "2024-12-13"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": true,
                        "first_data": "2024-12-13 15:38:51",
                        "last_data": "2024-12-13 15:47:08"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": true
                    }
                }
            },
            {
                "info": {
                    "id": 588645,
                    "location": "s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-23/",
                    "name": "dt",
                    "value": "2024-12-23"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": true,
                        "first_data": "2024-12-23 10:54:01",
                        "last_data": "2024-12-23 12:16:19"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": true
                    }
                }
            },
            {
                "info": {
                    "id": 588813,
                    "location": "s3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user/dt=2024-12-24/",
                    "name": "dt",
                    "value": "2024-12-24"
                },
                "state": {
                    "arrival": {
                        "athena_discovery": true,
                        "first_data": "2024-12-24 10:28:49",
                        "last_data": "2024-12-24 10:39:16"
                    },
                    "parquet": {
                        "first_performed": null,
                        "in_progress": null,
                        "job_id": null,
                        "last_performed": null,
                        "queued": true
                    }
                }
            }
        ],
        "type": "string"
    },
    "schema": {
        "attributes": {
            "_origin": "string",
            "channel": "string",
            "context.library.name": "string",
            "context.library.version": "string",
            "event": "string",
            "messageId": "string",
            "originalTimestamp": "string",
            "original_key": "string",
            "projectId": "string",
            "properties.errorMessage": "string",
            "properties.experimentList": [
                "string"
            ],
            "properties.experiments": "string",
            "properties.message": "string",
            "receivedAt": "string",
            "sentAt": "string",
            "timestamp": "string",
            "type": "string",
            "userId": "string",
            "version": "string",
            "writeKey": "string"
        },
        "created": "2019-03-16 00:30:41",
        "ddl": {
            "create_database": "CREATE DATABASE IF NOT EXISTS `sfdl_stage_segment`",
            "create_table": "CREATE EXTERNAL TABLE IF NOT EXISTS\n`sfdl_stage_segment.reactfoot_polluted_user`\n(\n`timestamp`STRING,`userId`STRING,`integrations`struct<`_empty_`:STRING>,`receivedAt`STRING,`original_key`STRING,`_origin`STRING,`properties`struct<`errorMessage`:STRING,`message`:STRING,`experiments`:STRING,`experimentList`:array>,`projectId`STRING,`event`STRING,`_metadata`struct<`nodeVersion`:STRING>,`version`STRING,`originalTimestamp`STRING,`context`struct<`library`:struct<`version`:STRING,`name`:STRING>>,`messageId`STRING,`writeKey`STRING,`type`STRING,`channel`STRING,`sentAt`STRING\n)\nPARTITIONED BY (dt string)\nROW FORMAT SERDE\n'org.openx.data.jsonserde.JsonSerDe'\n \nSTORED AS INPUTFORMAT\n  'org.apache.hadoop.mapred.TextInputFormat'\nOUTPUTFORMAT\n  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\nLOCATION\n  's3://sfdl-segment-reactfoot-stage/separated_dt/polluted_user'\n",
            "drop_table": "DROP TABLE IF EXISTS `sfdl_stage_segment.reactfoot_polluted_user`",
            "repair_table": "MSCK REPAIR TABLE sfdl_stage_segment.reactfoot_polluted_user"
        }
    },
    "server": {
        "hostname": "schemamanager-5c55c886bf-h2ts4",
        "name": "schema_manager",
        "process_start": "2025-10-21 13:03:27",
        "sf_env": "stage",
        "version": "2112174301"
    }
}