From ef6a4100918e5d46bbbc1005d076aaecbb3da8d0 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Thu, 22 Aug 2024 17:55:43 +0200 Subject: [PATCH] feat(ingest/s3): Partition support improvements (#11083) - Partition autodetection - Option to find min/max/min-max partition of a dataset - Generating Partition aspects --- metadata-ingestion/docs/sources/s3/s3.md | 33 +- .../source/data_lake_common/path_spec.py | 307 +- .../src/datahub/ingestion/source/s3/config.py | 5 + .../src/datahub/ingestion/source/s3/source.py | 382 ++- .../golden_mces_folder_no_partition.json | 440 +-- ...den_mces_folder_no_partition_filename.json | 440 +-- ...date_schema_with_partition_autodetect.json | 2572 +++++++++++++++++ ...ith_partition_autodetect_traverse_all.json | 2572 +++++++++++++++++ ...partition_autodetect_traverse_min_max.json | 2572 +++++++++++++++++ .../golden_mces_file_without_extension.json | 336 ++- .../s3/golden_mces_folder_no_partition.json | 317 +- ...lden_mces_folder_no_partition_exclude.json | 253 +- ...den_mces_folder_no_partition_filename.json | 317 +- .../golden_mces_folder_no_partition_glob.json | 253 +- .../golden_mces_folder_partition_basic.json | 265 +- .../golden_mces_folder_partition_keyval.json | 265 +- ...n_mces_folder_partition_update_schema.json | 265 +- ...date_schema_with_partition_autodetect.json | 1483 ++++++++++ ...ith_partition_autodetect_traverse_all.json | 1483 ++++++++++ ...partition_autodetect_traverse_min_max.json | 1483 ++++++++++ .../s3/golden_mces_multiple_files.json | 1256 ++++---- .../golden_mces_multiple_spec_for_files.json | 367 +-- ...s_multiple_specs_of_different_buckets.json | 532 ++-- .../s3/golden_mces_single_file.json | 244 +- .../s3/sources/s3/folder_no_partition.json | 1 + ...date_schema_with_partition_autodetect.json | 23 + ...ith_partition_autodetect_traverse_all.json | 24 + ...partition_autodetect_traverse_min_max.json | 24 + .../folder_aa/folder_aaa/food_csv/part3.csv | 5 + .../tests/integration/s3/test_s3.py | 48 + .../tests/unit/s3/test_s3_source.py | 73 +- 31 files changed, 15894 insertions(+), 2746 deletions(-) create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema_with_partition_autodetect.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_update_schema_with_partition_autodetect.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_all.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_min_max.json diff --git a/metadata-ingestion/docs/sources/s3/s3.md b/metadata-ingestion/docs/sources/s3/s3.md index 9484cd8de6666..90a2c30882d52 100644 --- a/metadata-ingestion/docs/sources/s3/s3.md +++ b/metadata-ingestion/docs/sources/s3/s3.md @@ -3,16 +3,31 @@ Path Specs (`path_specs`) is a list of Path Spec (`path_spec`) objects where each individual `path_spec` represents one or more datasets. Include path (`path_spec.include`) represents formatted path to the dataset. This path must end with `*.*` or `*.[ext]` to represent leaf level. If `*.[ext]` is provided then files with only specified extension type will be scanned. "`.[ext]`" can be any of [supported file types](#supported-file-types). Refer [example 1](#example-1---individual-file-as-dataset) below for more details. -All folder levels need to be specified in include path. You can use `/*/` to represent a folder level and avoid specifying exact folder name. To map folder as a dataset, use `{table}` placeholder to represent folder level for which dataset is to be created. For a partitioned dataset, you can use placeholder `{partition_key[i]}` to represent name of `i`th partition and `{partition[i]}` to represent value of `i`th partition. During ingestion, `i` will be used to match partition_key to partition. Refer [example 2 and 3](#example-2---folder-of-files-as-dataset-without-partitions) below for more details. +All folder levels need to be specified in include path. You can use `/*/` to represent a folder level and avoid specifying exact folder name. To map folder as a dataset, use `{table}` placeholder to represent folder level for which dataset is to be created. For a partitioned dataset, you can use placeholder `{partition_key[i]}` to represent name of `i`th partition and `{partition_value[i]}` to represent value of `i`th partition. During ingestion, `i` will be used to match partition_key to partition. Refer [example 2 and 3](#example-2---folder-of-files-as-dataset-without-partitions) below for more details. Exclude paths (`path_spec.exclude`) can be used to ignore paths that are not relevant to current `path_spec`. This path cannot have named variables ( `{}` ). Exclude path can have `**` to represent multiple folder levels. Refer [example 4](#example-4---folder-of-files-as-dataset-with-partitions-and-exclude-filter) below for more details. Refer [example 5](#example-5---advanced---either-individual-file-or-folder-of-files-as-dataset) if your bucket has more complex dataset representation. + **Additional points to note** - Folder names should not contain {, }, *, / in their names. - Named variable {folder} is reserved for internal working. please do not use in named variables. +#### Partitioned Dataset support +If your dataset is partitioned by the `partition_key`=`partition_value` format, then the partition values are auto-detected. + +Otherwise, you can specify partitions in the following way in the path_spec: +1. Specify partition_key and partition_value in the path like => `{partition_key[0]}={partition_value[0]}/{partition_key[1]}={partition_value[1]}/{partition_key[2]}={partition_value[2]}` +2. Partition key can be specify using named variables in the path_spec like => `year={year}/month={month}/day={day}` +3 if the path is in the form of /value1/value2/value3 the source infer partition value from the path and assign partition_0, partition_1, partition_2 etc + +Dataset creation time is determined by the creation time of earliest created file in the lowest partition while last updated time is determined by the last updated time of the latest updated file in the highest partition. + +How the source determines the highest/lowest partition it is based on the traversal method set in the path_spec. +- If the traversal method is set to `MAX` then the source will try to find the latest partition by ordering the partitions each level and find the latest partiton. This traversal method won't look for earilest partition/creation time but this is the fastest. +- If the traversal method is set to `MIN_MAX` then the source will try to find the latest and earliest partition by ordering the partitions each level and find the latest/earliest partiton. This traversal sort folders purely by name therefor it is fast but it doesn't guarantee the latest partition will have the latest created file. +- If the traversal method is set to `ALL` then the source will try to find the latest and earliest partition by listing all the files in all the partitions and find the creation/last modification time based on the file creations. This is the slowest but for non time partitioned datasets this is the only way to find the latest/earliest partition. ### Path Specs - Examples #### Example 1 - Individual file as Dataset @@ -73,7 +88,12 @@ test-bucket Path specs config to ingest folders `orders` and `returns` as datasets: ``` path_specs: - - include: s3://test-bucket/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.parquet + - include: s3://test-bucket/{table}/{partition_key[0]}={partition_value[0]}/{partition_key[1]}={partition_value[1]}/*.parquet +``` +or with partition auto-detection: +``` +path_specs: + - include: s3://test-bucket/{table}/ ``` One can also use `include: s3://test-bucket/{table}/*/*/*.parquet` here however above format is preferred as it allows declaring partitions explicitly. @@ -99,11 +119,15 @@ test-bucket Path specs config to ingest folder `orders` as dataset but not folder `tmp_orders`: ``` path_specs: - - include: s3://test-bucket/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.parquet + - include: s3://test-bucket/{table}/{partition_key[0]}={partition_value[0]}/{partition_key[1]}={partition_value[1]}/*.parquet exclude: - **/tmp_orders/** ``` - +or with partition auto-detection: +``` +path_specs: + - include: s3://test-bucket/{table}/ +``` #### Example 5 - Advanced - Either Individual file OR Folder of files as Dataset @@ -150,6 +174,7 @@ Above config has 3 path_specs and will ingest following datasets s3://my-bucket/foo/tests/bar.avro # single file table s3://my-bucket/foo/tests/*.* # mulitple file level tables s3://my-bucket/foo/tests/{table}/*.avro #table without partition +s3://my-bucket/foo/tests/{table}/ #table with partition autodetection. Partition only can be detected if it is in the format of key=value s3://my-bucket/foo/tests/{table}/*/*.avro #table where partitions are not specified s3://my-bucket/foo/tests/{table}/*.* # table where no partitions as well as data type specified s3://my-bucket/{dept}/tests/{table}/*.avro # specifying keywords to be used in display name diff --git a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py index 71765f9be5e32..4c2b0c276b9e7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py +++ b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py @@ -1,6 +1,8 @@ +import datetime import logging import os import re +from enum import Enum from typing import Any, Dict, List, Optional, Tuple, Union import parse @@ -28,6 +30,57 @@ "gzip", ] +java_to_python_mapping = { + "yyyy": "Y", + "MM": "m", + "dd": "d", + "HH": "H", + "mm": "M", + "ss": "S", +} + + +class SortKeyType(Enum): + STRING = "STRING" + INTEGER = "INTEGER" + FLOAT = "FLOAT" + DATETIME = "DATETIME" + DATE = "DATE" + + def __str__(self): + return self.value + + +class SortKey(ConfigModel): + key: str = Field( + description="The key to sort on. This can be a compound key based on the path_spec variables." + ) + type: SortKeyType = Field( + default=SortKeyType.STRING, + description="The date format to use when sorting. This is used to parse the date from the key. The format should follow the java [SimpleDateFormat](https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html) format.", + ) + + date_format: Optional[str] = Field( + default=None, + type=str, + description="The date format to use when sorting. This is used to parse the date from the key. The format should follow the java [SimpleDateFormat](https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html) format.", + ) + + @pydantic.validator("date_format", always=True) + def convert_date_format_to_python_format(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return None + else: + for java_format, python_format in java_to_python_mapping.items(): + v = v.replace(java_format, f"%{python_format}") + return v + + +class FolderTraversalMethod(Enum): + ALL = "ALL" + MIN_MAX = "MIN_MAX" + MAX = "MAX" + class PathSpec(ConfigModel): class Config: @@ -37,7 +90,7 @@ class Config: description="Path to table. Name variable `{table}` is used to mark the folder with dataset. In absence of `{table}`, file level dataset will be created. Check below examples for more details." ) exclude: Optional[List[str]] = Field( - default=None, + default=[], description="list of paths in glob pattern which will be excluded while scanning for the datasets", ) file_types: List[str] = Field( @@ -55,6 +108,13 @@ class Config: description="Display name of the dataset.Combination of named variables from include path and strings", ) + # This is not used yet, but will be used in the future to sort the partitions + sort_key: Optional[SortKey] = Field( + hidden_from_docs=True, + default=None, + description="Sort key to use when sorting the partitions. This is useful when the partitions are not sorted in the order of the data. The key can be a compound key based on the path_spec variables.", + ) + enable_compression: bool = Field( default=True, description="Enable or disable processing compressed files. Currently .gz and .bz files are supported.", @@ -70,8 +130,41 @@ class Config: description="Allow double stars in the include path. This can affect performance significantly if enabled", ) - def allowed(self, path: str) -> bool: + autodetect_partitions: bool = Field( + default=True, + description="Autodetect partition(s) from the path. If set to true, it will autodetect partition key/value if the folder format is {partition_key}={partition_value} for example `year=2024`", + ) + + traversal_method: FolderTraversalMethod = Field( + default=FolderTraversalMethod.MAX, + description="Method to traverse the folder. ALL: Traverse all the folders, MIN_MAX: Traverse the folders by finding min and max value, MAX: Traverse the folder with max value", + ) + + include_hidden_folders: bool = Field( + default=False, + description="Include hidden folders in the traversal (folders starting with . or _", + ) + + def is_path_hidden(self, path: str) -> bool: + # Split the path into directories and filename + dirs, filename = os.path.split(path) + + # Check the filename + if filename.startswith(".") or filename.startswith("_"): + return True + + # Check each directory in the path + for dir in dirs.split(os.sep): + if dir.startswith(".") or dir.startswith("_"): + return True + + return False + + def allowed(self, path: str, ignore_ext: bool = False) -> bool: logger.debug(f"Checking file to inclusion: {path}") + if self.is_path_hidden(path) and not self.include_hidden_folders: + return False + if not pathlib.PurePath(path).globmatch( self.glob_include, flags=pathlib.GLOBSTAR ): @@ -86,16 +179,20 @@ def allowed(self, path: str) -> bool: logger.debug(f"{path} is not excluded") ext = os.path.splitext(path)[1].strip(".") - if (ext == "" and self.default_extension is None) and ( - ext != "*" and ext not in self.file_types - ): - return False + if not ignore_ext: + if (ext == "" and self.default_extension is None) and ( + ext != "*" and ext not in self.file_types + ): + return False - logger.debug(f"{path} had selected extension {ext}") - logger.debug(f"{path} allowed for dataset creation") + logger.debug(f"{path} had selected extension {ext}") + logger.debug(f"{path} allowed for dataset creation") return True def dir_allowed(self, path: str) -> bool: + if self.glob_include.endswith("**"): + return self.allowed(path, ignore_ext=True) + path_slash = path.count("/") glob_slash = self.glob_include.count("/") if path_slash > glob_slash: @@ -126,13 +223,30 @@ def dir_allowed(self, path: str) -> bool: @classmethod def get_parsable_include(cls, include: str) -> str: parsable_include = include - for i in range(parsable_include.count("*")): - parsable_include = parsable_include.replace("*", f"{{folder[{i}]}}", 1) + if parsable_include.endswith("/{table}/**"): + # Remove the last two characters to make it parsable if it ends with {table}/** which marks autodetect partition + parsable_include = parsable_include[:-2] + else: + # Replace all * with {folder[i]} to make it parsable + for i in range(parsable_include.count("*")): + parsable_include = parsable_include.replace("*", f"{{folder[{i}]}}", 1) return parsable_include def get_named_vars(self, path: str) -> Union[None, parse.Result, parse.Match]: + if self.include.endswith("{table}/**"): + # If we have a partial path with ** at the end, we need to truncate the path to parse correctly + # parse needs to have exact number of folders to parse correctly and in case of ** we don't know the number of folders + # so we need to truncate the path to the last folder before ** to parse and get named vars correctly + splits = len(self.include[: self.include.find("{table}/")].split("/")) + path = "/".join(path.split("/", splits)[:-1]) + "/" + return self.compiled_include.parse(path) + def get_folder_named_vars( + self, path: str + ) -> Union[None, parse.Result, parse.Match]: + return self.compiled_folder_include.parse(path) + @pydantic.root_validator() def validate_no_double_stars(cls, values: Dict) -> Dict: if "include" not in values: @@ -227,6 +341,125 @@ def compiled_include(self): logger.debug(f"Setting compiled_include: {compiled_include}") return compiled_include + @cached_property + def compiled_folder_include(self): + parsable_folder_include = PathSpec.get_parsable_include(self.include).rsplit( + "/", 1 + )[0] + logger.debug(f"parsable_folder_include: {parsable_folder_include}") + compiled_folder_include = parse.compile(parsable_folder_include) + logger.debug(f"Setting compiled_folder_include: {compiled_folder_include}") + return compiled_folder_include + + @cached_property + def extract_variable_names(self): + # Regular expression to find all substrings enclosed in {} + pattern = r"\{(.*?)\}" + # Find all matches + matches = re.findall(pattern, self.include.split("{table}/")[1]) + return matches + + def get_partition_from_path(self, path: str) -> Optional[List[Tuple[str, str]]]: + # Automatic partition detection supports four methods to get partiton keys and values from path: + # Let's say we have the following path => year=2024/month=10/day=11 for this example you can specify the following path spec expressions: + # 1. User can specify partition_key and partition_value in the path like => {partition_key[0]}={partition_value[0]}/{partition_key[1]}={partition_value[1]}/{partition_key[2]}={partition_value[2]} + # 2. User can specify only partition key and the partition key will be used as partition name like => year={year}/month={month}/day={day} + # 3. You omit specifying anything and it will detect partiton key and value based on the equal signs (this only works if partitioned are specified in the key=value way. + # 4. if the path is in the form of /value1/value2/value3 we infer it from the path and assign partition_0, partition_1, partition_2 etc + + partition_keys: List[Tuple[str, str]] = [] + if self.include.find("{table}/"): + named_vars = self.get_named_vars(path) + if named_vars: + # If user has specified partition_key and partition_value in the path_spec then we use it to get partition keys + if "partition_key" in named_vars.named and ( + ( + "partition_value" in named_vars.named + and len(named_vars.named["partition_key"]) + == len(named_vars.named["partition_value"]) + ) + or ( + "partition" in named_vars.named + and len(named_vars.named["partition_key"]) + == len(named_vars.named["partition"]) + ) + ): + for key in named_vars.named["partition_key"]: + # We need to support both partition_value and partition as both were in our docs + if ( + "partition_value" in named_vars + and key in named_vars.named["partition_value"] + ) or ( + "partition" in named_vars + and key in named_vars.named["partition"] + ): + partition_keys.append( + ( + named_vars.named["partition_key"][key], + named_vars.named["partition_value"][key] + if "partition_value" in named_vars.named + else named_vars.named["partition"][key], + ) + ) + return partition_keys + else: + # TODO: Fix this message + logger.debug( + "Partition key or value not found. Fallbacking another mechanism to get partition keys" + ) + + partition_vars = self.extract_variable_names + if partition_vars: + for partition_key in partition_vars: + pkey: str = partition_key + index: Optional[int] = None + # We need to recreate the key and index from the partition_key + if partition_key.find("[") != -1: + pkey, index = partition_key.strip("]").split("[") + else: + pkey = partition_key + index = None + + if pkey in named_vars.named: + if index and index in named_vars.named[pkey]: + partition_keys.append( + (f"{pkey}_{index}", named_vars.named[pkey][index]) + ) + else: + partition_keys.append( + (partition_key, named_vars.named[partition_key]) + ) + return partition_keys + + # If user did not specified partition_key and partition_value in the path_spec then we use the default mechanism to get partition keys + if len(self.include.split("{table}/")) == 2: + num_slash = len(self.include.split("{table}/")[0].split("/")) + partition = path.split("/", num_slash)[num_slash] + else: + return None + if partition.endswith("/"): + partition = partition[:-1] + + # If partition is in the form of key=value we infer it from the path + if partition.find("=") != -1: + partition = partition.rsplit("/", 1)[0] + for partition_key in partition.split("/"): + if partition_key.find("=") != -1: + partition_keys.append(tuple(partition_key.split("="))) + else: + partition_split = partition.rsplit("/", 1) + if len(partition_split) == 1: + return None + partition = partition_split[0] + # If partition is in the form of /value1/value2/value3 we infer it from the path and assign partition_0, partition_1, partition_2 etc + num = 0 + for partition_value in partition.split("/"): + partition_keys.append((f"partition_{num}", partition_value)) + num += 1 + return partition_keys + + return None + @cached_property def glob_include(self): glob_include = re.sub(r"\{[^}]+\}", "*", self.include) @@ -244,7 +477,20 @@ def validate_path_spec(cls, values: Dict) -> Dict[str, Any]: ) return values + if values["include"] and values["autodetect_partitions"]: + include = values["include"] + if include.endswith("/"): + include = include[:-1] + + if include.endswith("{table}"): + values["include"] = include + "/**" + include_ext = os.path.splitext(values["include"])[1].strip(".") + if not include_ext: + include_ext = ( + "*" # if no extension is provided, we assume all files are allowed + ) + if ( include_ext not in values["file_types"] and include_ext != "*" @@ -263,6 +509,45 @@ def _extract_table_name(self, named_vars: dict) -> str: raise ValueError("path_spec.table_name is not set") return self.table_name.format_map(named_vars) + # TODO: Add support to sort partition folders by the defined partition key pattern. This is not implemented yet. + def extract_datetime_partition( + self, path: str, is_folder: bool = False + ) -> Optional[datetime.datetime]: + if self.sort_key is None: + return None + + if not self.sort_key.date_format and self.sort_key.type not in [ + SortKeyType.DATETIME, + SortKeyType.DATE, + ]: + return None + + if is_folder: + parsed_vars = self.get_folder_named_vars(path) + else: + parsed_vars = self.get_named_vars(path) + if parsed_vars is None: + return None + + partition_format = self.sort_key.key + datetime_format = self.sort_key.date_format + if datetime_format is None: + return None + + for var_key in parsed_vars.named: + var = parsed_vars.named[var_key] + if isinstance(var, dict): + for key in var: + template_key = var_key + f"[{key}]" + partition_format = partition_format.replace( + f"{{{template_key}}}", var[key] + ) + else: + partition_format.replace(f"{{{var_key}}}", var) + return datetime.datetime.strptime(partition_format, datetime_format).replace( + tzinfo=datetime.timezone.utc + ) + def extract_table_name_and_path(self, path: str) -> Tuple[str, str]: parsed_vars = self.get_named_vars(path) if parsed_vars is None or "table" not in parsed_vars.named: @@ -273,4 +558,4 @@ def extract_table_name_and_path(self, path: str) -> Tuple[str, str]: table_path = ( "/".join(path.split("/")[:depth]) + "/" + parsed_vars.named["table"] ) - return self._extract_table_name(parsed_vars.named), table_path + return self._extract_table_name(parsed_vars.named), table_path diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py index 921ab27564250..8f1b79251c466 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py @@ -98,6 +98,11 @@ class DataLakeSourceConfig( description="Whether to sort schema fields by fieldPath when inferring schemas.", ) + generate_partition_aspects: bool = Field( + default=True, + description="Whether to generate partition aspects for partitioned tables. On older servers for backward compatibility, this should be set to False. This flag will be removed in future versions.", + ) + def is_profiling_enabled(self) -> bool: return self.profiling.enabled and is_profiling_enabled( self.profiling.operation_config diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py index f81d06c35e3b0..55e25ebe88d12 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py @@ -5,10 +5,10 @@ import pathlib import re import time -from collections import OrderedDict from datetime import datetime +from itertools import groupby from pathlib import PurePath -from typing import Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple import smart_open.compression as so_compression from more_itertools import peekable @@ -43,6 +43,7 @@ strip_s3_prefix, ) from datahub.ingestion.source.data_lake_common.data_lake_utils import ContainerWUCreator +from datahub.ingestion.source.data_lake_common.path_spec import FolderTraversalMethod from datahub.ingestion.source.s3.config import DataLakeSourceConfig, PathSpec from datahub.ingestion.source.s3.report import DataLakeSourceReport from datahub.ingestion.source.schema_inference import avro, csv_tsv, json, parquet @@ -52,6 +53,7 @@ from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionSourceBase, ) +from datahub.metadata.com.linkedin.pegasus2avro.common import TimeStamp from datahub.metadata.com.linkedin.pegasus2avro.schema import ( SchemaField, SchemaMetadata, @@ -63,6 +65,8 @@ OperationClass, OperationTypeClass, OtherSchemaClass, + PartitionsSummaryClass, + PartitionSummaryClass, SchemaFieldDataTypeClass, _Aspect, ) @@ -134,16 +138,43 @@ def partitioned_folder_comparator(folder1: str, folder2: str) -> int: return 1 if folder1 > folder2 else -1 +@dataclasses.dataclass +class Folder: + creation_time: datetime + modification_time: datetime + size: int + sample_file: str + partition_id: Optional[List[Tuple[str, str]]] = None + is_partition: bool = False + + def partition_id_text(self) -> Optional[str]: + return ( + "/".join([f"{k}={v}" for k, v in self.partition_id]) + if self.partition_id + else None + ) + + +@dataclasses.dataclass +class BrowsePath: + file: str + timestamp: datetime + size: int + partitions: List[Folder] + + @dataclasses.dataclass class TableData: display_name: str is_s3: bool full_path: str - partitions: Optional[OrderedDict] timestamp: datetime table_path: str size_in_bytes: int number_of_files: int + partitions: Optional[List[Folder]] = None + max_partition: Optional[Folder] = None + min_partition: Optional[Folder] = None @platform_name("S3 / Local Files", id="s3") @@ -395,7 +426,7 @@ def get_fields(self, table_data: TableData, path_spec: PathSpec) -> List: if self.source_config.sort_schema_fields: fields = sorted(fields, key=lambda f: f.fieldPath) - if self.source_config.add_partition_columns_to_schema: + if self.source_config.add_partition_columns_to_schema and table_data.partitions: self.add_partition_columns_to_schema( fields=fields, path_spec=path_spec, full_path=table_data.full_path ) @@ -410,21 +441,23 @@ def add_partition_columns_to_schema( if field.fieldPath.startswith("[version=2.0]"): is_fieldpath_v2 = True break - vars = path_spec.get_named_vars(full_path) - if vars is not None and "partition_key" in vars: - for partition_key in vars["partition_key"].values(): - fields.append( - SchemaField( - fieldPath=f"{partition_key}" - if not is_fieldpath_v2 - else f"[version=2.0].[type=string].{partition_key}", - nativeDataType="string", - type=SchemaFieldDataTypeClass(StringTypeClass()), - isPartitioningKey=True, - nullable=True, - recursive=False, - ) + partition_keys = path_spec.get_partition_from_path(full_path) + if not partition_keys: + return None + + for partition_key in partition_keys: + fields.append( + SchemaField( + fieldPath=f"{partition_key[0]}" + if not is_fieldpath_v2 + else f"[version=2.0].[type=string].{partition_key[0]}", + nativeDataType="string", + type=SchemaFieldDataTypeClass(StringTypeClass()), + isPartitioningKey=True, + nullable=True, + recursive=False, ) + ) def get_table_profile( self, table_data: TableData, dataset_urn: str @@ -517,6 +550,39 @@ def _create_table_operation_aspect(self, table_data: TableData) -> OperationClas return operation + def __create_partition_summary_aspect( + self, partitions: List[Folder] + ) -> Optional[PartitionsSummaryClass]: + min_partition = min(partitions, key=lambda x: x.creation_time) + max_partition = max(partitions, key=lambda x: x.creation_time) + + max_partition_summary: Optional[PartitionSummaryClass] = None + + max_partition_id = max_partition.partition_id_text() + if max_partition_id is not None: + max_partition_summary = PartitionSummaryClass( + partition=max_partition_id, + createdTime=int(max_partition.creation_time.timestamp() * 1000), + lastModifiedTime=int( + max_partition.modification_time.timestamp() * 1000 + ), + ) + + min_partition_summary: Optional[PartitionSummaryClass] = None + min_partition_id = min_partition.partition_id_text() + if min_partition_id is not None: + min_partition_summary = PartitionSummaryClass( + partition=min_partition_id, + createdTime=int(min_partition.creation_time.timestamp() * 1000), + lastModifiedTime=int( + min_partition.modification_time.timestamp() * 1000 + ), + ) + + return PartitionsSummaryClass( + maxPartition=max_partition_summary, minPartition=min_partition_summary + ) + def ingest_table( self, table_data: TableData, path_spec: PathSpec ) -> Iterable[MetadataWorkUnit]: @@ -549,6 +615,12 @@ def ingest_table( customProperties = {"schema_inferred_from": str(table_data.full_path)} + min_partition: Optional[Folder] = None + max_partition: Optional[Folder] = None + if table_data.partitions: + min_partition = min(table_data.partitions, key=lambda x: x.creation_time) + max_partition = max(table_data.partitions, key=lambda x: x.creation_time) + if not path_spec.sample_files: customProperties.update( { @@ -556,11 +628,30 @@ def ingest_table( "size_in_bytes": str(table_data.size_in_bytes), } ) + else: + if table_data.partitions: + customProperties.update( + { + "number_of_partitions": str( + len(table_data.partitions) if table_data.partitions else 0 + ), + } + ) dataset_properties = DatasetPropertiesClass( description="", name=table_data.display_name, customProperties=customProperties, + created=( + TimeStamp(time=int(min_partition.creation_time.timestamp() * 1000)) + if min_partition + else None + ), + lastModified=( + TimeStamp(time=int(max_partition.modification_time.timestamp() * 1000)) + if max_partition + else None + ), ) aspects.append(dataset_properties) if table_data.size_in_bytes > 0: @@ -609,6 +700,12 @@ def ingest_table( operation = self._create_table_operation_aspect(table_data) aspects.append(operation) + + if table_data.partitions and self.source_config.generate_partition_aspects: + aspects.append( + self.__create_partition_summary_aspect(table_data.partitions) + ) + for mcp in MetadataChangeProposalWrapper.construct_many( entityUrn=dataset_urn, aspects=aspects, @@ -635,20 +732,35 @@ def extract_table_name(self, path_spec: PathSpec, named_vars: dict) -> str: return path_spec.table_name.format_map(named_vars) def extract_table_data( - self, path_spec: PathSpec, path: str, timestamp: datetime, size: int + self, + path_spec: PathSpec, + path: str, + timestamp: datetime, + size: int, + partitions: List[Folder], ) -> TableData: logger.debug(f"Getting table data for path: {path}") table_name, table_path = path_spec.extract_table_name_and_path(path) - table_data = None table_data = TableData( display_name=table_name, is_s3=self.is_s3_platform(), full_path=path, - partitions=None, + partitions=partitions, + max_partition=partitions[-1] if partitions else None, + min_partition=partitions[0] if partitions else None, timestamp=timestamp, table_path=table_path, number_of_files=1, - size_in_bytes=size, + size_in_bytes=( + size + if size + else sum( + [ + partition.size if partition.size else 0 + for partition in partitions + ] + ) + ), ) return table_data @@ -668,8 +780,17 @@ def resolve_templated_folders(self, bucket_name: str, prefix: str) -> Iterable[s ) def get_dir_to_process( - self, bucket_name: str, folder: str, path_spec: PathSpec, protocol: str - ) -> str: + self, + bucket_name: str, + folder: str, + path_spec: PathSpec, + protocol: str, + min: bool = False, + ) -> List[str]: + + # if len(path_spec.include.split("/")) == len(f"{protocol}{bucket_name}/{folder}".split("/")): + # return [f"{protocol}{bucket_name}/{folder}"] + iterator = list_folders( bucket_name=bucket_name, prefix=folder, @@ -680,23 +801,102 @@ def get_dir_to_process( sorted_dirs = sorted( iterator, key=functools.cmp_to_key(partitioned_folder_comparator), - reverse=True, + reverse=not min, ) + folders = [] for dir in sorted_dirs: if path_spec.dir_allowed(f"{protocol}{bucket_name}/{dir}/"): - return self.get_dir_to_process( + folders_list = self.get_dir_to_process( bucket_name=bucket_name, folder=dir + "/", path_spec=path_spec, protocol=protocol, + min=min, ) - return folder - else: - return folder + folders.extend(folders_list) + if not path_spec.traversal_method == FolderTraversalMethod.ALL: + return folders + if folders: + return folders + else: + return [f"{protocol}{bucket_name}/{folder}"] + return [f"{protocol}{bucket_name}/{folder}"] + + def get_folder_info( + self, + path_spec: PathSpec, + bucket: Any, # Todo: proper type + prefix: str, + ) -> List[Folder]: + """ + Retrieves all the folders in a path by listing all the files in the prefix. + If the prefix is a full path then only that folder will be extracted. + + A folder has creation and modification times, size, and a sample file path. + - Creation time is the earliest creation time of all files in the folder. + - Modification time is the latest modification time of all files in the folder. + - Size is the sum of all file sizes in the folder. + - Sample file path is used for schema inference later. (sample file is the latest created file in the folder) + + Parameters: + path_spec (PathSpec): The path specification used to determine partitioning. + bucket (Any): The S3 bucket object. + prefix (str): The prefix path in the S3 bucket to list objects from. + + Returns: + List[Folder]: A list of Folder objects representing the partitions found. + """ + + prefix_to_list = prefix + files = list( + bucket.objects.filter(Prefix=f"{prefix_to_list}").page_size(PAGE_SIZE) + ) + files = sorted(files, key=lambda a: a.last_modified) + grouped_files = groupby(files, lambda x: x.key.rsplit("/", 1)[0]) + + partitions: List[Folder] = [] + for key, group in grouped_files: + file_size = 0 + creation_time = None + modification_time = None + + for item in group: + file_path = self.create_s3_path(item.bucket_name, item.key) + if not path_spec.allowed(file_path): + logger.debug(f"File {file_path} not allowed and skipping") + continue + file_size += item.size + if creation_time is None or item.last_modified < creation_time: + creation_time = item.last_modified + if modification_time is None or item.last_modified > modification_time: + modification_time = item.last_modified + max_file = item + + if modification_time is None: + logger.warning( + f"Unable to find any files in the folder {key}. Skipping..." + ) + continue + + id = path_spec.get_partition_from_path( + self.create_s3_path(max_file.bucket_name, max_file.key) + ) + + # If id is None, it means the folder is not a partition + partitions.append( + Folder( + partition_id=id, + is_partition=bool(id), + creation_time=creation_time if creation_time else None, + modification_time=modification_time, + sample_file=self.create_s3_path(max_file.bucket_name, max_file.key), + size=file_size, + ) + ) + + return partitions - def s3_browser( - self, path_spec: PathSpec, sample_size: int - ) -> Iterable[Tuple[str, datetime, int]]: + def s3_browser(self, path_spec: PathSpec, sample_size: int) -> Iterable[BrowsePath]: if self.source_config.aws_config is None: raise ValueError("aws_config not set. Cannot browse s3") s3 = self.source_config.aws_config.get_s3_resource( @@ -728,6 +928,9 @@ def s3_browser( include = include.replace(max_match, "*") max_start = match.start() max_match = match.group() + # We stop at {table} + if max_match == "{table}": + break table_index = include.find(max_match) for folder in self.resolve_templated_folders( @@ -737,24 +940,68 @@ def s3_browser( for f in list_folders( bucket_name, f"{folder}", self.source_config.aws_config ): + dirs_to_process = [] logger.info(f"Processing folder: {f}") - protocol = ContainerWUCreator.get_protocol(path_spec.include) - dir_to_process = self.get_dir_to_process( - bucket_name=bucket_name, - folder=f + "/", - path_spec=path_spec, - protocol=protocol, + if path_spec.traversal_method == FolderTraversalMethod.ALL: + dirs_to_process.append(f) + else: + if ( + path_spec.traversal_method + == FolderTraversalMethod.MIN_MAX + or path_spec.traversal_method + == FolderTraversalMethod.MAX + ): + protocol = ContainerWUCreator.get_protocol( + path_spec.include + ) + dirs_to_process_max = self.get_dir_to_process( + bucket_name=bucket_name, + folder=f + "/", + path_spec=path_spec, + protocol=protocol, + ) + dirs_to_process.append(dirs_to_process_max[0]) + + if ( + path_spec.traversal_method + == FolderTraversalMethod.MIN_MAX + ): + dirs_to_process_min = self.get_dir_to_process( + bucket_name=bucket_name, + folder=f + "/", + path_spec=path_spec, + protocol=protocol, + min=True, + ) + dirs_to_process.append(dirs_to_process_min[0]) + folders = [] + for dir in dirs_to_process: + logger.info(f"Getting files from folder: {dir}") + prefix_to_process = dir.rstrip("\\").lstrip( + self.create_s3_path(bucket_name, "/") + ) + + folders.extend( + self.get_folder_info( + path_spec, bucket, prefix_to_process + ) + ) + max_folder = None + if folders: + max_folder = max(folders, key=lambda x: x.modification_time) + if not max_folder: + logger.warning( + f"Unable to find any files in the folder {dir}. Skipping..." + ) + continue + + partitions = list(filter(lambda x: x.is_partition, folders)) + yield BrowsePath( + file=max_folder.sample_file, + timestamp=max_folder.modification_time, + size=max_folder.size, + partitions=partitions, ) - logger.info(f"Getting files from folder: {dir_to_process}") - dir_to_process = dir_to_process.rstrip("\\") - for obj in ( - bucket.objects.filter(Prefix=f"{dir_to_process}") - .page_size(PAGE_SIZE) - .limit(sample_size) - ): - s3_path = self.create_s3_path(obj.bucket_name, obj.key) - logger.debug(f"Sampling file: {s3_path}") - yield s3_path, obj.last_modified, obj.size, except Exception as e: # This odd check if being done because boto does not have a proper exception to catch # The exception that appears in stacktrace cannot actually be caught without a lot more work @@ -774,18 +1021,26 @@ def s3_browser( for obj in bucket.objects.filter(Prefix=prefix).page_size(PAGE_SIZE): s3_path = self.create_s3_path(obj.bucket_name, obj.key) logger.debug(f"Path: {s3_path}") - yield s3_path, obj.last_modified, obj.size, + yield BrowsePath( + file=s3_path, + timestamp=obj.last_modified, + size=obj.size, + partitions=[], + ) def create_s3_path(self, bucket_name: str, key: str) -> str: return f"s3://{bucket_name}/{key}" - def local_browser(self, path_spec: PathSpec) -> Iterable[Tuple[str, datetime, int]]: + def local_browser(self, path_spec: PathSpec) -> Iterable[BrowsePath]: prefix = self.get_prefix(path_spec.include) if os.path.isfile(prefix): logger.debug(f"Scanning single local file: {prefix}") - yield prefix, datetime.utcfromtimestamp( - os.path.getmtime(prefix) - ), os.path.getsize(prefix) + yield BrowsePath( + file=prefix, + timestamp=datetime.utcfromtimestamp(os.path.getmtime(prefix)), + size=os.path.getsize(prefix), + partitions=[], + ) else: logger.debug(f"Scanning files under local folder: {prefix}") for root, dirs, files in os.walk(prefix): @@ -796,9 +1051,14 @@ def local_browser(self, path_spec: PathSpec) -> Iterable[Tuple[str, datetime, in full_path = PurePath( os.path.normpath(os.path.join(root, file)) ).as_posix() - yield full_path, datetime.utcfromtimestamp( - os.path.getmtime(full_path) - ), os.path.getsize(full_path) + yield BrowsePath( + file=full_path, + timestamp=datetime.utcfromtimestamp( + os.path.getmtime(full_path) + ), + size=os.path.getsize(full_path), + partitions=[], + ) def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.container_WU_creator = ContainerWUCreator( @@ -817,11 +1077,15 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: else self.local_browser(path_spec) ) table_dict: Dict[str, TableData] = {} - for file, timestamp, size in file_browser: - if not path_spec.allowed(file): + for browse_path in file_browser: + if not path_spec.allowed(browse_path.file): continue table_data = self.extract_table_data( - path_spec, file, timestamp, size + path_spec, + browse_path.file, + browse_path.timestamp, + browse_path.size, + browse_path.partitions, ) if table_data.table_path not in table_dict: table_dict[table_data.table_path] = table_data diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json index 51a8ef60849a9..03d2ac38cf7d2 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json @@ -7,9 +7,9 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part2.csv", + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv", "number_of_files": "3", - "size_in_bytes": "3446" + "size_in_bytes": "3539" }, "name": "food_csv", "description": "", @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -58,18 +59,6 @@ "recursive": false, "isPartOfKey": false }, - { - "fieldPath": "healthy", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "isPartOfKey": false - }, { "fieldPath": "height", "nullable": false, @@ -111,7 +100,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -123,8 +113,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445089021 @@ -132,7 +122,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -152,7 +143,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -167,7 +159,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -182,7 +175,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -199,7 +193,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -214,7 +209,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -234,7 +230,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -249,7 +246,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -264,7 +262,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -281,7 +280,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -296,7 +296,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -316,7 +317,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -336,7 +338,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -351,7 +354,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -366,7 +370,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -383,7 +388,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -398,7 +404,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -422,7 +429,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -442,7 +450,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -457,7 +466,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -472,7 +482,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -489,7 +500,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -504,7 +516,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -532,7 +545,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -552,7 +566,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +582,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +598,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +616,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +632,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +665,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -666,7 +686,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +702,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +718,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +736,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +752,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -764,7 +789,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -784,7 +810,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -799,7 +826,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -814,7 +842,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -831,7 +860,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -846,7 +876,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -886,7 +917,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -906,7 +938,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +954,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +970,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -953,7 +988,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -968,7 +1004,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1012,7 +1049,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1027,7 +1065,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1039,299 +1078,127 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "rowCount": 70, - "columnCount": 5, + "rowCount": 4, + "columnCount": 4, "fieldProfiles": [ { "fieldPath": "name", - "uniqueCount": 9, - "uniqueProportion": 0.13043478260869565, - "nullCount": 1, - "nullProportion": 0.014285714285714285, + "uniqueCount": 4, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, "distinctValueFrequencies": [ - { - "value": "NullValue", - "frequency": 1 - }, { "value": "apple", - "frequency": 7 - }, - { - "value": "chicken", - "frequency": 7 + "frequency": 1 }, { "value": "cookie", - "frequency": 6 - }, - { - "value": "hamburger", - "frequency": 7 + "frequency": 1 }, { "value": "lasagna", - "frequency": 7 - }, - { - "value": "orange", - "frequency": 7 + "frequency": 1 }, { "value": "pasta", - "frequency": 7 - }, - { - "value": "spinach", - "frequency": 7 - }, - { - "value": "sushi", - "frequency": 7 - }, - { - "value": "water", - "frequency": 7 + "frequency": 1 } ], "sampleValues": [ "apple", - "apple", - "apple", - "chicken", - "cookie", "cookie", - "cookie", - "lasagna", "lasagna", - "orange", - "orange", - "pasta", - "pasta", - "pasta", - "pasta", - "spinach", - "spinach", - "spinach", - "water", - "water" + "pasta" ] }, { "fieldPath": "weight", - "uniqueCount": 9, - "uniqueProportion": 0.12857142857142856, + "uniqueCount": 4, + "uniqueProportion": 1.0, "nullCount": 0, "nullProportion": 0.0, "distinctValueFrequencies": [ - { - "value": "10", - "frequency": 7 - }, - { - "value": "2", - "frequency": 7 - }, { "value": "23", - "frequency": 7 - }, - { - "value": "32", - "frequency": 7 - }, - { - "value": "36", - "frequency": 7 - }, - { - "value": "43", - "frequency": 7 + "frequency": 1 }, { "value": "49", - "frequency": 7 + "frequency": 1 }, { "value": "50", - "frequency": 7 - }, - { - "value": "53", - "frequency": 7 + "frequency": 1 }, { "value": "72", - "frequency": 7 + "frequency": 1 } ], "sampleValues": [ - "10", - "10", - "10", - "23", - "23", "23", - "32", - "32", - "36", - "43", - "43", - "49", "49", "50", - "50", - "50", - "72", - "72", - "72", "72" ] }, { "fieldPath": "height", - "uniqueCount": 4, - "uniqueProportion": 0.05714285714285714, + "uniqueCount": 1, + "uniqueProportion": 0.25, "nullCount": 0, "nullProportion": 0.0, "distinctValueFrequencies": [ { "value": "4", - "frequency": 24 - }, - { - "value": "5", - "frequency": 15 - }, - { - "value": "6", - "frequency": 23 - }, - { - "value": "7", - "frequency": 8 + "frequency": 4 } ], "sampleValues": [ "4", "4", "4", - "4", - "4", - "4", - "4", - "5", - "5", - "5", - "5", - "5", - "6", - "6", - "6", - "6", - "6", - "6", - "7", - "7" + "4" ] }, { "fieldPath": "color", - "uniqueCount": 7, - "uniqueProportion": 0.1, + "uniqueCount": 3, + "uniqueProportion": 0.75, "nullCount": 0, "nullProportion": 0.0, "distinctValueFrequencies": [ - { - "value": "blue", - "frequency": 7 - }, { "value": "brown", - "frequency": 14 - }, - { - "value": "green", - "frequency": 7 - }, - { - "value": "orange", - "frequency": 14 + "frequency": 1 }, { "value": "red", - "frequency": 14 - }, - { - "value": "white", - "frequency": 7 + "frequency": 2 }, { "value": "yellow", - "frequency": 7 + "frequency": 1 } ], "sampleValues": [ - "blue", - "blue", - "brown", - "brown", "brown", - "green", - "green", - "green", - "orange", - "orange", - "red", - "red", "red", "red", - "red", - "white", - "yellow", - "yellow", - "yellow", "yellow" ] - }, - { - "fieldPath": "healthy", - "uniqueCount": 2, - "uniqueProportion": 0.028985507246376812, - "nullCount": 1, - "nullProportion": 0.014285714285714285, - "sampleValues": [ - "False", - "False", - "False", - "False", - "False", - "False", - "False", - "False", - "False", - "None", - "True", - "True", - "True", - "True", - "True", - "True", - "True", - "True", - "True", - "True" - ] } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1379,7 +1246,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1401,7 +1269,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1494,7 +1363,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1506,8 +1376,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445119021 @@ -1515,7 +1385,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1530,7 +1401,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1542,8 +1414,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 70, "columnCount": 5, @@ -1834,7 +1706,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1882,7 +1755,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1897,7 +1771,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1912,7 +1787,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json index 8e67bbf10bd6d..48c59753fcaff 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json @@ -7,9 +7,9 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part2.csv", + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv", "number_of_files": "3", - "size_in_bytes": "3446" + "size_in_bytes": "3539" }, "name": "folder_aaa.food_csv", "description": "", @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -58,18 +59,6 @@ "recursive": false, "isPartOfKey": false }, - { - "fieldPath": "healthy", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "isPartOfKey": false - }, { "fieldPath": "height", "nullable": false, @@ -111,7 +100,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -123,8 +113,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445089021 @@ -132,7 +122,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -152,7 +143,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -167,7 +159,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -182,7 +175,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -199,7 +193,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -214,7 +209,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -234,7 +230,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -249,7 +246,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -264,7 +262,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -281,7 +280,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -296,7 +296,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -316,7 +317,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -336,7 +338,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -351,7 +354,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -366,7 +370,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -383,7 +388,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -398,7 +404,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -422,7 +429,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -442,7 +450,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -457,7 +466,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -472,7 +482,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -489,7 +500,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -504,7 +516,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -532,7 +545,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -552,7 +566,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +582,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +598,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -599,7 +616,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -614,7 +632,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +665,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -666,7 +686,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -681,7 +702,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -696,7 +718,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -713,7 +736,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -728,7 +752,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -764,7 +789,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -784,7 +810,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -799,7 +826,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -814,7 +842,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -831,7 +860,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -846,7 +876,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -886,7 +917,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -906,7 +938,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -921,7 +954,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -936,7 +970,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -953,7 +988,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -968,7 +1004,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1012,7 +1049,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1027,7 +1065,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1039,299 +1078,127 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "rowCount": 70, - "columnCount": 5, + "rowCount": 4, + "columnCount": 4, "fieldProfiles": [ { "fieldPath": "name", - "uniqueCount": 9, - "uniqueProportion": 0.13043478260869565, - "nullCount": 1, - "nullProportion": 0.014285714285714285, + "uniqueCount": 4, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, "distinctValueFrequencies": [ - { - "value": "NullValue", - "frequency": 1 - }, { "value": "apple", - "frequency": 7 - }, - { - "value": "chicken", - "frequency": 7 + "frequency": 1 }, { "value": "cookie", - "frequency": 6 - }, - { - "value": "hamburger", - "frequency": 7 + "frequency": 1 }, { "value": "lasagna", - "frequency": 7 - }, - { - "value": "orange", - "frequency": 7 + "frequency": 1 }, { "value": "pasta", - "frequency": 7 - }, - { - "value": "spinach", - "frequency": 7 - }, - { - "value": "sushi", - "frequency": 7 - }, - { - "value": "water", - "frequency": 7 + "frequency": 1 } ], "sampleValues": [ "apple", - "apple", - "apple", - "chicken", - "cookie", "cookie", - "cookie", - "lasagna", "lasagna", - "orange", - "orange", - "pasta", - "pasta", - "pasta", - "pasta", - "spinach", - "spinach", - "spinach", - "water", - "water" + "pasta" ] }, { "fieldPath": "weight", - "uniqueCount": 9, - "uniqueProportion": 0.12857142857142856, + "uniqueCount": 4, + "uniqueProportion": 1.0, "nullCount": 0, "nullProportion": 0.0, "distinctValueFrequencies": [ - { - "value": "10", - "frequency": 7 - }, - { - "value": "2", - "frequency": 7 - }, { "value": "23", - "frequency": 7 - }, - { - "value": "32", - "frequency": 7 - }, - { - "value": "36", - "frequency": 7 - }, - { - "value": "43", - "frequency": 7 + "frequency": 1 }, { "value": "49", - "frequency": 7 + "frequency": 1 }, { "value": "50", - "frequency": 7 - }, - { - "value": "53", - "frequency": 7 + "frequency": 1 }, { "value": "72", - "frequency": 7 + "frequency": 1 } ], "sampleValues": [ - "10", - "10", - "10", - "23", - "23", "23", - "32", - "32", - "36", - "43", - "43", - "49", "49", "50", - "50", - "50", - "72", - "72", - "72", "72" ] }, { "fieldPath": "height", - "uniqueCount": 4, - "uniqueProportion": 0.05714285714285714, + "uniqueCount": 1, + "uniqueProportion": 0.25, "nullCount": 0, "nullProportion": 0.0, "distinctValueFrequencies": [ { "value": "4", - "frequency": 24 - }, - { - "value": "5", - "frequency": 15 - }, - { - "value": "6", - "frequency": 23 - }, - { - "value": "7", - "frequency": 8 + "frequency": 4 } ], "sampleValues": [ "4", "4", "4", - "4", - "4", - "4", - "4", - "5", - "5", - "5", - "5", - "5", - "6", - "6", - "6", - "6", - "6", - "6", - "7", - "7" + "4" ] }, { "fieldPath": "color", - "uniqueCount": 7, - "uniqueProportion": 0.1, + "uniqueCount": 3, + "uniqueProportion": 0.75, "nullCount": 0, "nullProportion": 0.0, "distinctValueFrequencies": [ - { - "value": "blue", - "frequency": 7 - }, { "value": "brown", - "frequency": 14 - }, - { - "value": "green", - "frequency": 7 - }, - { - "value": "orange", - "frequency": 14 + "frequency": 1 }, { "value": "red", - "frequency": 14 - }, - { - "value": "white", - "frequency": 7 + "frequency": 2 }, { "value": "yellow", - "frequency": 7 + "frequency": 1 } ], "sampleValues": [ - "blue", - "blue", - "brown", - "brown", "brown", - "green", - "green", - "green", - "orange", - "orange", - "red", - "red", "red", "red", - "red", - "white", - "yellow", - "yellow", - "yellow", "yellow" ] - }, - { - "fieldPath": "healthy", - "uniqueCount": 2, - "uniqueProportion": 0.028985507246376812, - "nullCount": 1, - "nullProportion": 0.014285714285714285, - "sampleValues": [ - "False", - "False", - "False", - "False", - "False", - "False", - "False", - "False", - "False", - "None", - "True", - "True", - "True", - "True", - "True", - "True", - "True", - "True", - "True", - "True" - ] } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1379,7 +1246,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1401,7 +1269,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1494,7 +1363,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1506,8 +1376,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1688445119021 @@ -1515,7 +1385,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1530,7 +1401,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1542,8 +1414,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 70, "columnCount": 5, @@ -1834,7 +1706,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1882,7 +1755,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1897,7 +1771,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1912,7 +1787,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema_with_partition_autodetect.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema_with_partition_autodetect.json new file mode 100644 index 0000000000000..adb3686309e6c --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_update_schema_with_partition_autodetect.json @@ -0,0 +1,2572 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv", + "number_of_files": "3", + "size_in_bytes": "3539" + }, + "name": "folder_aaa.food_csv", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_csv", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833420000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests" + }, + "name": "tests" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration" + }, + "name": "integration" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3" + }, + "name": "s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data" + }, + "name": "test_data" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system" + }, + "name": "local_system" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" + }, + "name": "folder_a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" + }, + "name": "folder_aaa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:98a716614da5246426edd48260406364" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 4, + "columnCount": 4, + "fieldProfiles": [ + { + "fieldPath": "name", + "uniqueCount": 4, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "apple", + "frequency": 1 + }, + { + "value": "cookie", + "frequency": 1 + }, + { + "value": "lasagna", + "frequency": 1 + }, + { + "value": "pasta", + "frequency": 1 + } + ], + "sampleValues": [ + "apple", + "cookie", + "lasagna", + "pasta" + ] + }, + { + "fieldPath": "weight", + "uniqueCount": 4, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "23", + "frequency": 1 + }, + { + "value": "49", + "frequency": 1 + }, + { + "value": "50", + "frequency": 1 + }, + { + "value": "72", + "frequency": 1 + } + ], + "sampleValues": [ + "23", + "49", + "50", + "72" + ] + }, + { + "fieldPath": "height", + "uniqueCount": 1, + "uniqueProportion": 0.25, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "4", + "frequency": 4 + } + ], + "sampleValues": [ + "4", + "4", + "4", + "4" + ] + }, + { + "fieldPath": "color", + "uniqueCount": 3, + "uniqueProportion": 0.75, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "brown", + "frequency": 1 + }, + { + "value": "red", + "frequency": 2 + }, + { + "value": "yellow", + "frequency": 1 + } + ], + "sampleValues": [ + "brown", + "red", + "red", + "yellow" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet", + "number_of_files": "2", + "size_in_bytes": "8412" + }, + "name": "folder_aaa.food_parquet", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_parquet", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "healthy", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833440000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 70, + "columnCount": 5, + "fieldProfiles": [ + { + "fieldPath": "name", + "uniqueCount": 9, + "uniqueProportion": 0.13043478260869565, + "nullCount": 1, + "nullProportion": 0.014285714285714285, + "distinctValueFrequencies": [ + { + "value": "NullValue", + "frequency": 1 + }, + { + "value": "apple", + "frequency": 7 + }, + { + "value": "chicken", + "frequency": 7 + }, + { + "value": "cookie", + "frequency": 6 + }, + { + "value": "hamburger", + "frequency": 7 + }, + { + "value": "lasagna", + "frequency": 7 + }, + { + "value": "orange", + "frequency": 7 + }, + { + "value": "pasta", + "frequency": 7 + }, + { + "value": "spinach", + "frequency": 7 + }, + { + "value": "sushi", + "frequency": 7 + }, + { + "value": "water", + "frequency": 7 + } + ], + "sampleValues": [ + "apple", + "apple", + "apple", + "chicken", + "cookie", + "cookie", + "cookie", + "lasagna", + "lasagna", + "orange", + "orange", + "pasta", + "pasta", + "pasta", + "pasta", + "spinach", + "spinach", + "spinach", + "water", + "water" + ] + }, + { + "fieldPath": "weight", + "uniqueCount": 10, + "uniqueProportion": 0.14285714285714285, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "10", + "frequency": 7 + }, + { + "value": "2", + "frequency": 7 + }, + { + "value": "23", + "frequency": 7 + }, + { + "value": "32", + "frequency": 7 + }, + { + "value": "36", + "frequency": 7 + }, + { + "value": "43", + "frequency": 7 + }, + { + "value": "49", + "frequency": 7 + }, + { + "value": "50", + "frequency": 7 + }, + { + "value": "53", + "frequency": 7 + }, + { + "value": "72", + "frequency": 7 + } + ], + "sampleValues": [ + "10", + "10", + "10", + "23", + "23", + "23", + "32", + "32", + "36", + "43", + "43", + "49", + "49", + "50", + "50", + "50", + "72", + "72", + "72", + "72" + ] + }, + { + "fieldPath": "height", + "uniqueCount": 4, + "uniqueProportion": 0.05714285714285714, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "4", + "frequency": 24 + }, + { + "value": "5", + "frequency": 15 + }, + { + "value": "6", + "frequency": 23 + }, + { + "value": "7", + "frequency": 8 + } + ], + "sampleValues": [ + "4", + "4", + "4", + "4", + "4", + "4", + "4", + "5", + "5", + "5", + "5", + "5", + "6", + "6", + "6", + "6", + "6", + "6", + "7", + "7" + ] + }, + { + "fieldPath": "color", + "uniqueCount": 7, + "uniqueProportion": 0.1, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "blue", + "frequency": 7 + }, + { + "value": "brown", + "frequency": 14 + }, + { + "value": "green", + "frequency": 7 + }, + { + "value": "orange", + "frequency": 14 + }, + { + "value": "red", + "frequency": 14 + }, + { + "value": "white", + "frequency": 7 + }, + { + "value": "yellow", + "frequency": 7 + } + ], + "sampleValues": [ + "blue", + "blue", + "brown", + "brown", + "brown", + "green", + "green", + "green", + "orange", + "orange", + "red", + "red", + "red", + "red", + "red", + "white", + "yellow", + "yellow", + "yellow", + "yellow" + ] + }, + { + "fieldPath": "healthy", + "uniqueCount": 2, + "uniqueProportion": 0.028985507246376812, + "nullCount": 1, + "nullProportion": 0.014285714285714285, + "sampleValues": [ + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "None", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_files": "13", + "size_in_bytes": "188600" + }, + "name": "folder_aaa.pokemon_abilities_json", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.is_native", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.short_effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "is_main_series", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.is_hidden", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.slot", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833590000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 1, + "columnCount": 9, + "fieldProfiles": [ + { + "fieldPath": "effect_changes", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(is_native='no', name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(is_native='yes', name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]" + ] + }, + { + "fieldPath": "effect_entries", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\n\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\n\\nAu\u00dferhalb vom Kampf: Wenn ein Pok\u00e9mon mit dieser F\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\u00fcckschrecken zu lassen.'), Row(effect=\"This Pok\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\n\\nThis ability does not stack with a held item.\\n\\nOverworld: The wild encounter rate is halved while this Pok\u00e9mon is first in the party.\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\u00e9mon flinch with each hit.')]" + ] + }, + { + "fieldPath": "flavor_text_entries", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\neffrayer l\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\u304f\u3055\u304f\u3066\\u3000\u3042\u3044\u3066\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\uc545\ucde8 \ub54c\ubb38\uc5d0 \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\nl\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\u00e4sst den Gegner durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\u81ed\u304f\u3066\\u3000\u76f8\u624b\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\u304f\u3055\u304f\u3066\\u3000\u3042\u3044\u3066\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\uc545\ucde8 \ub54c\ubb38\uc5d0 \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\nl\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\u00e4sst den Gegner durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\u81ed\u304f\u3066\\u3000\u76f8\u624b\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\u304f\u3055\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u306f\u306a\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u3053\u3046\u3052\u304d\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u3042\u3044\u3066\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\uc545\ucde8\ub97c \ud48d\uaca8\uc11c\\n\uacf5\uaca9\ud588\uc744 \ub54c \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u767c\u51fa\u81ed\u6c23\uff0c\\n\u5728\u653b\u64ca\u7684\u6642\u5019\uff0c\\n\u6709\u6642\u6703\u4f7f\u5c0d\u624b\u754f\u7e2e\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\u00e9mon \u00e9met une odeur si naus\u00e9abonde\\nqu\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\u00e9mon\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\u00e9mon\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u81ed\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u653e\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u653b\u6483\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u76f8\u624b\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u901a\u8fc7\u91ca\u653e\u81ed\u81ed\u7684\u6c14\u5473\uff0c\\n\u5728\u653b\u51fb\u7684\u65f6\u5019\uff0c\\n\u6709\u65f6\u4f1a\u4f7f\u5bf9\u624b\u754f\u7f29\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u304f\u3055\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u306f\u306a\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u3053\u3046\u3052\u304d\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u3042\u3044\u3066\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\uc545\ucde8\ub97c \ud48d\uaca8\uc11c\\n\uacf5\uaca9\ud588\uc744 \ub54c \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u767c\u51fa\u81ed\u6c23\uff0c\\n\u5728\u653b\u64ca\u7684\u6642\u5019\uff0c\\n\u6709\u6642\u6703\u4f7f\u5c0d\u624b\u754f\u7e2e\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\u00e9mon \u00e9met une odeur si naus\u00e9abonde\\nqu\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\u00e9mon\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\u00e9mon\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u81ed\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u653e\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u653b\u6483\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u76f8\u624b\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u901a\u8fc7\u91ca\u653e\u81ed\u81ed\u7684\u6c14\u5473\uff0c\\n\u5728\u653b\u51fb\u7684\u65f6\u5019\uff0c\\n\u6709\u65f6\u4f1a\u4f7f\u5bf9\u624b\u754f\u7f29\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]" + ] + }, + { + "fieldPath": "generation", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')" + ] + }, + { + "fieldPath": "id", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "1", + "frequency": 1 + } + ], + "sampleValues": [ + "1" + ] + }, + { + "fieldPath": "is_main_series", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "True" + ] + }, + { + "fieldPath": "name", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "stench", + "frequency": 1 + } + ], + "sampleValues": [ + "stench" + ] + }, + { + "fieldPath": "names", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\u3042\u304f\u3057\u3085\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\uc545\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\u60e1\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\u3042\u304f\u3057\u3085\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\u6076\u81ed')]" + ] + }, + { + "fieldPath": "pokemon", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json new file mode 100644 index 0000000000000..80f584788fdb2 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json @@ -0,0 +1,2572 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv", + "number_of_files": "3", + "size_in_bytes": "3539" + }, + "name": "folder_aaa.food_csv", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_csv", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833420000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests" + }, + "name": "tests" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration" + }, + "name": "integration" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3" + }, + "name": "s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data" + }, + "name": "test_data" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system" + }, + "name": "local_system" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" + }, + "name": "folder_a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" + }, + "name": "folder_aaa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:98a716614da5246426edd48260406364" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 4, + "columnCount": 4, + "fieldProfiles": [ + { + "fieldPath": "name", + "uniqueCount": 4, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "apple", + "frequency": 1 + }, + { + "value": "cookie", + "frequency": 1 + }, + { + "value": "lasagna", + "frequency": 1 + }, + { + "value": "pasta", + "frequency": 1 + } + ], + "sampleValues": [ + "apple", + "cookie", + "lasagna", + "pasta" + ] + }, + { + "fieldPath": "weight", + "uniqueCount": 4, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "23", + "frequency": 1 + }, + { + "value": "49", + "frequency": 1 + }, + { + "value": "50", + "frequency": 1 + }, + { + "value": "72", + "frequency": 1 + } + ], + "sampleValues": [ + "23", + "49", + "50", + "72" + ] + }, + { + "fieldPath": "height", + "uniqueCount": 1, + "uniqueProportion": 0.25, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "4", + "frequency": 4 + } + ], + "sampleValues": [ + "4", + "4", + "4", + "4" + ] + }, + { + "fieldPath": "color", + "uniqueCount": 3, + "uniqueProportion": 0.75, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "brown", + "frequency": 1 + }, + { + "value": "red", + "frequency": 2 + }, + { + "value": "yellow", + "frequency": 1 + } + ], + "sampleValues": [ + "brown", + "red", + "red", + "yellow" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet", + "number_of_files": "2", + "size_in_bytes": "8412" + }, + "name": "folder_aaa.food_parquet", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_parquet", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "healthy", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833440000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 70, + "columnCount": 5, + "fieldProfiles": [ + { + "fieldPath": "name", + "uniqueCount": 9, + "uniqueProportion": 0.13043478260869565, + "nullCount": 1, + "nullProportion": 0.014285714285714285, + "distinctValueFrequencies": [ + { + "value": "NullValue", + "frequency": 1 + }, + { + "value": "apple", + "frequency": 7 + }, + { + "value": "chicken", + "frequency": 7 + }, + { + "value": "cookie", + "frequency": 6 + }, + { + "value": "hamburger", + "frequency": 7 + }, + { + "value": "lasagna", + "frequency": 7 + }, + { + "value": "orange", + "frequency": 7 + }, + { + "value": "pasta", + "frequency": 7 + }, + { + "value": "spinach", + "frequency": 7 + }, + { + "value": "sushi", + "frequency": 7 + }, + { + "value": "water", + "frequency": 7 + } + ], + "sampleValues": [ + "apple", + "apple", + "apple", + "chicken", + "cookie", + "cookie", + "cookie", + "lasagna", + "lasagna", + "orange", + "orange", + "pasta", + "pasta", + "pasta", + "pasta", + "spinach", + "spinach", + "spinach", + "water", + "water" + ] + }, + { + "fieldPath": "weight", + "uniqueCount": 10, + "uniqueProportion": 0.14285714285714285, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "10", + "frequency": 7 + }, + { + "value": "2", + "frequency": 7 + }, + { + "value": "23", + "frequency": 7 + }, + { + "value": "32", + "frequency": 7 + }, + { + "value": "36", + "frequency": 7 + }, + { + "value": "43", + "frequency": 7 + }, + { + "value": "49", + "frequency": 7 + }, + { + "value": "50", + "frequency": 7 + }, + { + "value": "53", + "frequency": 7 + }, + { + "value": "72", + "frequency": 7 + } + ], + "sampleValues": [ + "10", + "10", + "10", + "23", + "23", + "23", + "32", + "32", + "36", + "43", + "43", + "49", + "49", + "50", + "50", + "50", + "72", + "72", + "72", + "72" + ] + }, + { + "fieldPath": "height", + "uniqueCount": 4, + "uniqueProportion": 0.05714285714285714, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "4", + "frequency": 24 + }, + { + "value": "5", + "frequency": 15 + }, + { + "value": "6", + "frequency": 23 + }, + { + "value": "7", + "frequency": 8 + } + ], + "sampleValues": [ + "4", + "4", + "4", + "4", + "4", + "4", + "4", + "5", + "5", + "5", + "5", + "5", + "6", + "6", + "6", + "6", + "6", + "6", + "7", + "7" + ] + }, + { + "fieldPath": "color", + "uniqueCount": 7, + "uniqueProportion": 0.1, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "blue", + "frequency": 7 + }, + { + "value": "brown", + "frequency": 14 + }, + { + "value": "green", + "frequency": 7 + }, + { + "value": "orange", + "frequency": 14 + }, + { + "value": "red", + "frequency": 14 + }, + { + "value": "white", + "frequency": 7 + }, + { + "value": "yellow", + "frequency": 7 + } + ], + "sampleValues": [ + "blue", + "blue", + "brown", + "brown", + "brown", + "green", + "green", + "green", + "orange", + "orange", + "red", + "red", + "red", + "red", + "red", + "white", + "yellow", + "yellow", + "yellow", + "yellow" + ] + }, + { + "fieldPath": "healthy", + "uniqueCount": 2, + "uniqueProportion": 0.028985507246376812, + "nullCount": 1, + "nullProportion": 0.014285714285714285, + "sampleValues": [ + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "None", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_files": "13", + "size_in_bytes": "188600" + }, + "name": "folder_aaa.pokemon_abilities_json", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.is_native", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.short_effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "is_main_series", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.is_hidden", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.slot", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833590000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 1, + "columnCount": 9, + "fieldProfiles": [ + { + "fieldPath": "effect_changes", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(is_native='no', name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(is_native='yes', name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]" + ] + }, + { + "fieldPath": "effect_entries", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\n\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\n\\nAu\u00dferhalb vom Kampf: Wenn ein Pok\u00e9mon mit dieser F\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\u00fcckschrecken zu lassen.'), Row(effect=\"This Pok\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\n\\nThis ability does not stack with a held item.\\n\\nOverworld: The wild encounter rate is halved while this Pok\u00e9mon is first in the party.\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\u00e9mon flinch with each hit.')]" + ] + }, + { + "fieldPath": "flavor_text_entries", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\neffrayer l\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\u304f\u3055\u304f\u3066\\u3000\u3042\u3044\u3066\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\uc545\ucde8 \ub54c\ubb38\uc5d0 \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\nl\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\u00e4sst den Gegner durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\u81ed\u304f\u3066\\u3000\u76f8\u624b\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\u304f\u3055\u304f\u3066\\u3000\u3042\u3044\u3066\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\uc545\ucde8 \ub54c\ubb38\uc5d0 \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\nl\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\u00e4sst den Gegner durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\u81ed\u304f\u3066\\u3000\u76f8\u624b\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\u304f\u3055\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u306f\u306a\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u3053\u3046\u3052\u304d\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u3042\u3044\u3066\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\uc545\ucde8\ub97c \ud48d\uaca8\uc11c\\n\uacf5\uaca9\ud588\uc744 \ub54c \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u767c\u51fa\u81ed\u6c23\uff0c\\n\u5728\u653b\u64ca\u7684\u6642\u5019\uff0c\\n\u6709\u6642\u6703\u4f7f\u5c0d\u624b\u754f\u7e2e\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\u00e9mon \u00e9met une odeur si naus\u00e9abonde\\nqu\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\u00e9mon\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\u00e9mon\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u81ed\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u653e\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u653b\u6483\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u76f8\u624b\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u901a\u8fc7\u91ca\u653e\u81ed\u81ed\u7684\u6c14\u5473\uff0c\\n\u5728\u653b\u51fb\u7684\u65f6\u5019\uff0c\\n\u6709\u65f6\u4f1a\u4f7f\u5bf9\u624b\u754f\u7f29\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u304f\u3055\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u306f\u306a\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u3053\u3046\u3052\u304d\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u3042\u3044\u3066\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\uc545\ucde8\ub97c \ud48d\uaca8\uc11c\\n\uacf5\uaca9\ud588\uc744 \ub54c \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u767c\u51fa\u81ed\u6c23\uff0c\\n\u5728\u653b\u64ca\u7684\u6642\u5019\uff0c\\n\u6709\u6642\u6703\u4f7f\u5c0d\u624b\u754f\u7e2e\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\u00e9mon \u00e9met une odeur si naus\u00e9abonde\\nqu\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\u00e9mon\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\u00e9mon\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u81ed\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u653e\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u653b\u6483\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u76f8\u624b\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u901a\u8fc7\u91ca\u653e\u81ed\u81ed\u7684\u6c14\u5473\uff0c\\n\u5728\u653b\u51fb\u7684\u65f6\u5019\uff0c\\n\u6709\u65f6\u4f1a\u4f7f\u5bf9\u624b\u754f\u7f29\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]" + ] + }, + { + "fieldPath": "generation", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')" + ] + }, + { + "fieldPath": "id", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "1", + "frequency": 1 + } + ], + "sampleValues": [ + "1" + ] + }, + { + "fieldPath": "is_main_series", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "True" + ] + }, + { + "fieldPath": "name", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "stench", + "frequency": 1 + } + ], + "sampleValues": [ + "stench" + ] + }, + { + "fieldPath": "names", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\u3042\u304f\u3057\u3085\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\uc545\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\u60e1\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\u3042\u304f\u3057\u3085\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\u6076\u81ed')]" + ] + }, + { + "fieldPath": "pokemon", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json new file mode 100644 index 0000000000000..a384a8f1e501d --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json @@ -0,0 +1,2572 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv", + "number_of_files": "3", + "size_in_bytes": "3539" + }, + "name": "folder_aaa.food_csv", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_csv", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833420000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests" + }, + "name": "tests" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration" + }, + "name": "integration" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3" + }, + "name": "s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data" + }, + "name": "test_data" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system" + }, + "name": "local_system" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a" + }, + "name": "folder_a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "file", + "env": "UAT", + "folder_abs_path": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa" + }, + "name": "folder_aaa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:file" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:98a716614da5246426edd48260406364" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 4, + "columnCount": 4, + "fieldProfiles": [ + { + "fieldPath": "name", + "uniqueCount": 4, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "apple", + "frequency": 1 + }, + { + "value": "cookie", + "frequency": 1 + }, + { + "value": "lasagna", + "frequency": 1 + }, + { + "value": "pasta", + "frequency": 1 + } + ], + "sampleValues": [ + "apple", + "cookie", + "lasagna", + "pasta" + ] + }, + { + "fieldPath": "weight", + "uniqueCount": 4, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "23", + "frequency": 1 + }, + { + "value": "49", + "frequency": 1 + }, + { + "value": "50", + "frequency": 1 + }, + { + "value": "72", + "frequency": 1 + } + ], + "sampleValues": [ + "23", + "49", + "50", + "72" + ] + }, + { + "fieldPath": "height", + "uniqueCount": 1, + "uniqueProportion": 0.25, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "4", + "frequency": 4 + } + ], + "sampleValues": [ + "4", + "4", + "4", + "4" + ] + }, + { + "fieldPath": "color", + "uniqueCount": 3, + "uniqueProportion": 0.75, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "brown", + "frequency": 1 + }, + { + "value": "red", + "frequency": 2 + }, + { + "value": "yellow", + "frequency": 1 + } + ], + "sampleValues": [ + "brown", + "red", + "red", + "yellow" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet", + "number_of_files": "2", + "size_in_bytes": "8412" + }, + "name": "folder_aaa.food_parquet", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_parquet", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "healthy", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833440000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 70, + "columnCount": 5, + "fieldProfiles": [ + { + "fieldPath": "name", + "uniqueCount": 9, + "uniqueProportion": 0.13043478260869565, + "nullCount": 1, + "nullProportion": 0.014285714285714285, + "distinctValueFrequencies": [ + { + "value": "NullValue", + "frequency": 1 + }, + { + "value": "apple", + "frequency": 7 + }, + { + "value": "chicken", + "frequency": 7 + }, + { + "value": "cookie", + "frequency": 6 + }, + { + "value": "hamburger", + "frequency": 7 + }, + { + "value": "lasagna", + "frequency": 7 + }, + { + "value": "orange", + "frequency": 7 + }, + { + "value": "pasta", + "frequency": 7 + }, + { + "value": "spinach", + "frequency": 7 + }, + { + "value": "sushi", + "frequency": 7 + }, + { + "value": "water", + "frequency": 7 + } + ], + "sampleValues": [ + "apple", + "apple", + "apple", + "chicken", + "cookie", + "cookie", + "cookie", + "lasagna", + "lasagna", + "orange", + "orange", + "pasta", + "pasta", + "pasta", + "pasta", + "spinach", + "spinach", + "spinach", + "water", + "water" + ] + }, + { + "fieldPath": "weight", + "uniqueCount": 10, + "uniqueProportion": 0.14285714285714285, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "10", + "frequency": 7 + }, + { + "value": "2", + "frequency": 7 + }, + { + "value": "23", + "frequency": 7 + }, + { + "value": "32", + "frequency": 7 + }, + { + "value": "36", + "frequency": 7 + }, + { + "value": "43", + "frequency": 7 + }, + { + "value": "49", + "frequency": 7 + }, + { + "value": "50", + "frequency": 7 + }, + { + "value": "53", + "frequency": 7 + }, + { + "value": "72", + "frequency": 7 + } + ], + "sampleValues": [ + "10", + "10", + "10", + "23", + "23", + "23", + "32", + "32", + "36", + "43", + "43", + "49", + "49", + "50", + "50", + "50", + "72", + "72", + "72", + "72" + ] + }, + { + "fieldPath": "height", + "uniqueCount": 4, + "uniqueProportion": 0.05714285714285714, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "4", + "frequency": 24 + }, + { + "value": "5", + "frequency": 15 + }, + { + "value": "6", + "frequency": 23 + }, + { + "value": "7", + "frequency": 8 + } + ], + "sampleValues": [ + "4", + "4", + "4", + "4", + "4", + "4", + "4", + "5", + "5", + "5", + "5", + "5", + "6", + "6", + "6", + "6", + "6", + "6", + "7", + "7" + ] + }, + { + "fieldPath": "color", + "uniqueCount": 7, + "uniqueProportion": 0.1, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "blue", + "frequency": 7 + }, + { + "value": "brown", + "frequency": 14 + }, + { + "value": "green", + "frequency": 7 + }, + { + "value": "orange", + "frequency": 14 + }, + { + "value": "red", + "frequency": 14 + }, + { + "value": "white", + "frequency": 7 + }, + { + "value": "yellow", + "frequency": 7 + } + ], + "sampleValues": [ + "blue", + "blue", + "brown", + "brown", + "brown", + "green", + "green", + "green", + "orange", + "orange", + "red", + "red", + "red", + "red", + "red", + "white", + "yellow", + "yellow", + "yellow", + "yellow" + ] + }, + { + "fieldPath": "healthy", + "uniqueCount": 2, + "uniqueProportion": 0.028985507246376812, + "nullCount": 1, + "nullProportion": 0.014285714285714285, + "sampleValues": [ + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "False", + "None", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True", + "True" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_files": "13", + "size_in_bytes": "188600" + }, + "name": "folder_aaa.pokemon_abilities_json", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.is_native", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.short_effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "is_main_series", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.is_hidden", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.slot", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586833590000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 1, + "columnCount": 9, + "fieldProfiles": [ + { + "fieldPath": "effect_changes", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(is_native='no', name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(is_native='yes', name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]" + ] + }, + { + "fieldPath": "effect_entries", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\n\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\n\\nAu\u00dferhalb vom Kampf: Wenn ein Pok\u00e9mon mit dieser F\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\u00fcckschrecken zu lassen.'), Row(effect=\"This Pok\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\n\\nThis ability does not stack with a held item.\\n\\nOverworld: The wild encounter rate is halved while this Pok\u00e9mon is first in the party.\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\u00e9mon flinch with each hit.')]" + ] + }, + { + "fieldPath": "flavor_text_entries", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\nwild Pok\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\neffrayer l\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\u304f\u3055\u304f\u3066\\u3000\u3042\u3044\u3066\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\uc545\ucde8 \ub54c\ubb38\uc5d0 \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\nl\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\u00e4sst den Gegner durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\u81ed\u304f\u3066\\u3000\u76f8\u624b\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\u304f\u3055\u304f\u3066\\u3000\u3042\u3044\u3066\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\uc545\ucde8 \ub54c\ubb38\uc5d0 \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\nl\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\u00e4sst den Gegner durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\u81ed\u304f\u3066\\u3000\u76f8\u624b\u304c\\n\u3072\u308b\u3080\\u3000\u3053\u3068\u304c\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\u304f\u3055\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u306f\u306a\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u3053\u3046\u3052\u304d\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u3042\u3044\u3066\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\uc545\ucde8\ub97c \ud48d\uaca8\uc11c\\n\uacf5\uaca9\ud588\uc744 \ub54c \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u767c\u51fa\u81ed\u6c23\uff0c\\n\u5728\u653b\u64ca\u7684\u6642\u5019\uff0c\\n\u6709\u6642\u6703\u4f7f\u5c0d\u624b\u754f\u7e2e\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\u00e9mon \u00e9met une odeur si naus\u00e9abonde\\nqu\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\u00e9mon\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\u00e9mon\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u81ed\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u653e\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u653b\u6483\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u76f8\u624b\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u901a\u8fc7\u91ca\u653e\u81ed\u81ed\u7684\u6c14\u5473\uff0c\\n\u5728\u653b\u51fb\u7684\u65f6\u5019\uff0c\\n\u6709\u65f6\u4f1a\u4f7f\u5bf9\u624b\u754f\u7f29\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\u304f\u3055\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u306f\u306a\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u3053\u3046\u3052\u304d\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u3042\u3044\u3066\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\uc545\ucde8\ub97c \ud48d\uaca8\uc11c\\n\uacf5\uaca9\ud588\uc744 \ub54c \uc0c1\ub300\uac00\\n\ud480\uc8fd\uc744 \ub54c\uac00 \uc788\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u767c\u51fa\u81ed\u6c23\uff0c\\n\u5728\u653b\u64ca\u7684\u6642\u5019\uff0c\\n\u6709\u6642\u6703\u4f7f\u5c0d\u624b\u754f\u7e2e\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\u00e9mon \u00e9met une odeur si naus\u00e9abonde\\nqu\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\nzur\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\u00e9mon\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\u00e9mon\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u81ed\u3044\\u3000\u306b\u304a\u3044\u3092\\u3000\u653e\u3064\u3053\u3068\u306b\u3088\u3063\u3066\\n\u653b\u6483\u3057\u305f\\u3000\u3068\u304d\u306b\\u3000\u76f8\u624b\u3092\\n\u3072\u308b\u307e\u305b\u308b\u3053\u3068\u304c\\u3000\u3042\u308b\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\u901a\u8fc7\u91ca\u653e\u81ed\u81ed\u7684\u6c14\u5473\uff0c\\n\u5728\u653b\u51fb\u7684\u65f6\u5019\uff0c\\n\u6709\u65f6\u4f1a\u4f7f\u5bf9\u624b\u754f\u7f29\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]" + ] + }, + { + "fieldPath": "generation", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')" + ] + }, + { + "fieldPath": "id", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "1", + "frequency": 1 + } + ], + "sampleValues": [ + "1" + ] + }, + { + "fieldPath": "is_main_series", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "True" + ] + }, + { + "fieldPath": "name", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "stench", + "frequency": 1 + } + ], + "sampleValues": [ + "stench" + ] + }, + { + "fieldPath": "names", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\u3042\u304f\u3057\u3085\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\uc545\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\u60e1\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\u3042\u304f\u3057\u3085\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\u6076\u81ed')]" + ] + }, + { + "fieldPath": "pokemon", + "uniqueCount": 1, + "uniqueProportion": 1.0, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "urn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc" + }, + { + "id": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "urn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886" + }, + { + "id": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "urn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4" + }, + { + "id": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "urn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50" + }, + { + "id": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "urn": "urn:li:container:93525defb812252106d3b0c08a55e39a" + }, + { + "id": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "urn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156" + }, + { + "id": "urn:li:container:98a716614da5246426edd48260406364", + "urn": "urn:li:container:98a716614da5246426edd48260406364" + }, + { + "id": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "urn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json index d50f00efacaa0..06678512199bf 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json @@ -12,7 +12,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -34,7 +35,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -127,54 +129,51 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/no_extension/small,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "containerProperties", "aspect": { "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "customProperties": { + "platform": "s3", + "instance": "test-platform-instance", + "env": "DEV", + "folder_abs_path": "my-test-bucket/folder_a" }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586848010000 + "name": "folder_a" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "container", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "instance": "test-platform-instance", - "env": "DEV", - "bucket_name": "my-test-bucket" - }, - "name": "my-test-bucket" + "container": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -184,96 +183,103 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/no_extension/small,DEV)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847850000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "S3 bucket" + "Folder" ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "containerProperties", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - } - ] + "customProperties": { + "platform": "s3", + "instance": "test-platform-instance", + "env": "DEV", + "bucket_name": "my-test-bucket" + }, + "name": "my-test-bucket" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "instance": "test-platform-instance", - "env": "DEV", - "folder_abs_path": "my-test-bucket/folder_a" - }, - "name": "folder_a" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:s3", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -289,39 +295,47 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Folder" + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + "typeNames": [ + "S3 bucket" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -345,12 +359,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -359,50 +374,52 @@ "platform": "s3", "instance": "test-platform-instance", "env": "DEV", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "name": "folder_aa" + "name": "folder_aaa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -414,86 +431,63 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "containerProperties", "aspect": { "json": { - "container": "urn:li:container:c8d940d2010edd365619411b385b11e4" + "customProperties": { + "platform": "s3", + "instance": "test-platform-instance", + "env": "DEV", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - }, - { - "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", - "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" - }, - { - "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", - "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "file_without_extension.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", - "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "instance": "test-platform-instance", - "env": "DEV", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" - }, - "name": "folder_aaa" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:s3", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -509,12 +503,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -526,22 +521,37 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + } + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -573,7 +583,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:c8d940d2010edd365619411b385b11e4" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -594,38 +621,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:9b4624d58669059c9e62afb3d7341944", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:9b4624d58669059c9e62afb3d7341944", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -642,22 +671,25 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:9b4624d58669059c9e62afb3d7341944", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + "platform": "urn:li:dataPlatform:s3", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -693,7 +725,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -708,7 +741,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -723,7 +757,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } }, { @@ -763,7 +798,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "file_without_extension.json" + "runId": "file_without_extension.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json index 0eefc4cafbd62..8e1daed0070f1 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part2.csv" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part3.csv" }, "name": "food_csv", "description": "", @@ -16,7 +16,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -56,18 +57,6 @@ "recursive": false, "isPartOfKey": false }, - { - "fieldPath": "healthy", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "isPartOfKey": false - }, { "fieldPath": "height", "nullable": false, @@ -109,28 +98,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847990000 - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -150,7 +119,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -165,22 +135,30 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847820000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -197,7 +175,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -212,7 +207,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -232,7 +228,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -247,7 +244,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -262,39 +260,47 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "containerProperties", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -314,27 +320,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "container", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -349,22 +352,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -381,22 +386,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -420,7 +427,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -440,37 +448,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -487,22 +498,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -530,7 +543,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -545,39 +559,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", - "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" - }, - { - "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", - "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" - }, - { - "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", - "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" - }, - { - "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", - "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" - } - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -597,7 +596,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -690,7 +690,41 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -702,16 +736,17 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847810000 + "lastUpdatedTimestamp": 1586847840000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -726,7 +761,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } }, { @@ -758,37 +810,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_no_partition.json" + "runId": "folder_no_partition.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json index 36f40cc918735..0d4c2d14d4ac1 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json @@ -16,7 +16,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -109,28 +110,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847810000 - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -150,7 +131,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -165,22 +147,30 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847840000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -197,206 +187,219 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "path": [] + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a" - }, - "name": "folder_a" + "path": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "containerProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "containerProperties", "aspect": { "json": { - "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", - "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" - } - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "subTypes", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Folder" + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -420,7 +423,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -440,37 +460,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -487,22 +510,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -530,37 +555,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "status", "aspect": { "json": { - "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } }, { @@ -592,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_exclude.json" + "runId": "folder_no_partition_exclude.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json index 7a59a7f21f118..7172bba73f01c 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part2.csv" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part3.csv" }, "name": "folder_aaa.food_csv", "description": "", @@ -16,7 +16,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -56,18 +57,6 @@ "recursive": false, "isPartOfKey": false }, - { - "fieldPath": "healthy", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "isPartOfKey": false - }, { "fieldPath": "height", "nullable": false, @@ -109,28 +98,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847990000 - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -150,7 +119,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -165,22 +135,30 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847820000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -197,7 +175,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -212,7 +207,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -232,7 +228,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -247,7 +244,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -262,39 +260,47 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "containerProperties", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -314,27 +320,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "container", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -349,22 +352,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -381,22 +386,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -420,7 +427,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -440,37 +448,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -487,22 +498,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -530,7 +543,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -545,39 +559,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", - "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" - }, - { - "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", - "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" - }, - { - "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", - "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" - }, - { - "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", - "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" - } - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -597,7 +596,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -690,7 +690,41 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -702,16 +736,17 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847810000 + "lastUpdatedTimestamp": 1586847840000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -726,7 +761,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } }, { @@ -758,37 +810,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_no_partition_filename.json" + "runId": "folder_no_partition_filename.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json index 0dc71b900b03b..0372a6afd9457 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json @@ -16,7 +16,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -109,28 +110,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847810000 - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -150,7 +131,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -165,22 +147,30 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847840000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -197,206 +187,219 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "path": [] + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a" - }, - "name": "folder_a" + "path": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "containerProperties", "aspect": { "json": { - "removed": false + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "containerProperties", "aspect": { "json": { - "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", - "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" - } - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "subTypes", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Folder" + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -420,7 +423,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -440,37 +460,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -487,22 +510,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -530,37 +555,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "status", "aspect": { "json": { - "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } }, { @@ -592,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_no_partition_glob.json" + "runId": "folder_no_partition_glob.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json index aa683a2bcd4c2..6ff3925b1afb3 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json @@ -7,16 +7,24 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_partitions": "1" }, "name": "folder_aaa.pokemon_abilities_json", "description": "", + "created": { + "time": 1586847980000 + }, + "lastModified": { + "time": 1586847980000 + }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -541,48 +549,54 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "containerProperties", "aspect": { "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "customProperties": { + "platform": "s3", + "env": "UAT", + "bucket_name": "my-test-bucket" }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847960000 + "name": "my-test-bucket" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "partitionsSummary", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "bucket_name": "my-test-bucket" + "minPartition": { + "partition": "partition_0=year=2022/partition_1=month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 }, - "name": "my-test-bucket" + "maxPartition": { + "partition": "partition_0=year=2022/partition_1=month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 + } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -597,22 +611,30 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847990000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -629,7 +651,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -644,12 +683,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -657,49 +697,52 @@ "customProperties": { "platform": "s3", "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a" + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_a" + "name": "folder_aa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -711,77 +754,61 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "containerProperties", "aspect": { "json": { - "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", - "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", - "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -796,12 +823,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -813,22 +841,29 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -852,7 +887,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -872,37 +908,56 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -919,22 +974,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -962,7 +1019,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -977,7 +1035,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -992,7 +1051,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1024,7 +1084,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_basic.json" + "runId": "folder_partition_basic.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json index 095c1cbe3152b..86fa7835008dd 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json @@ -7,16 +7,24 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_partitions": "1" }, "name": "folder_aaa.pokemon_abilities_json", "description": "", + "created": { + "time": 1586847980000 + }, + "lastModified": { + "time": 1586847980000 + }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -541,48 +549,54 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "containerProperties", "aspect": { "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "customProperties": { + "platform": "s3", + "env": "UAT", + "bucket_name": "my-test-bucket" }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847960000 + "name": "my-test-bucket" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "partitionsSummary", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "bucket_name": "my-test-bucket" + "minPartition": { + "partition": "year=2022/month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 }, - "name": "my-test-bucket" + "maxPartition": { + "partition": "year=2022/month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 + } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -597,22 +611,30 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847990000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -629,7 +651,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -644,12 +683,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -657,49 +697,52 @@ "customProperties": { "platform": "s3", "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a" + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_a" + "name": "folder_aa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -711,77 +754,61 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "containerProperties", "aspect": { "json": { - "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", - "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", - "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -796,12 +823,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -813,22 +841,29 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -852,7 +887,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -872,37 +908,56 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -919,22 +974,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -962,7 +1019,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -977,7 +1035,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -992,7 +1051,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1024,7 +1084,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_keyval.json" + "runId": "folder_partition_keyval.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json index a5f135c30369a..95ec5e83f7991 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json @@ -7,16 +7,24 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_partitions": "1" }, "name": "folder_aaa.pokemon_abilities_json", "description": "", + "created": { + "time": 1586847980000 + }, + "lastModified": { + "time": 1586847980000 + }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -541,48 +549,54 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "containerProperties", "aspect": { "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "customProperties": { + "platform": "s3", + "env": "UAT", + "bucket_name": "my-test-bucket" }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847960000 + "name": "my-test-bucket" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "partitionsSummary", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "bucket_name": "my-test-bucket" + "minPartition": { + "partition": "partition_0=year=2022/partition_1=month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 }, - "name": "my-test-bucket" + "maxPartition": { + "partition": "partition_0=year=2022/partition_1=month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 + } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -597,22 +611,30 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847990000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -629,7 +651,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -644,12 +683,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -657,49 +697,52 @@ "customProperties": { "platform": "s3", "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a" + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_a" + "name": "folder_aa" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -711,77 +754,61 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "containerProperties", "aspect": { "json": { - "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", - "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", - "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "UAT", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -796,12 +823,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -813,22 +841,29 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -852,7 +887,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -872,37 +908,56 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -919,22 +974,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -962,7 +1019,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -977,7 +1035,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -992,7 +1051,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1024,7 +1084,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "folder_partition_update_schema.json" + "runId": "folder_partition_update_schema.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json new file mode 100644 index 0000000000000..4fee8fb36b13f --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json @@ -0,0 +1,1483 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part3.csv" + }, + "name": "folder_aaa.food_csv", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_csv", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "bucket_name": "my-test-bucket" + }, + "name": "my-test-bucket" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847820000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "S3 bucket" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" + }, + "name": "folder_aaa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" + }, + "name": "folder_aaa.food_parquet", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_parquet", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "healthy", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847840000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_partitions": "1" + }, + "name": "folder_aaa.pokemon_abilities_json", + "description": "", + "created": { + "time": 1586847980000 + }, + "lastModified": { + "time": 1586847980000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.is_native", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.short_effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "is_main_series", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.is_hidden", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.slot", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "partitionsSummary", + "aspect": { + "json": { + "minPartition": { + "partition": "year=2022/month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 + }, + "maxPartition": { + "partition": "year=2022/month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847990000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_update_schema_with_partition_autodetect.json", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json new file mode 100644 index 0000000000000..13ba57f5be671 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json @@ -0,0 +1,1483 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part3.csv" + }, + "name": "folder_aaa.food_csv", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_csv", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "bucket_name": "my-test-bucket" + }, + "name": "my-test-bucket" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847820000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "S3 bucket" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" + }, + "name": "folder_aaa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" + }, + "name": "folder_aaa.food_parquet", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_parquet", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "healthy", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847840000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_partitions": "7" + }, + "name": "folder_aaa.pokemon_abilities_json", + "description": "", + "created": { + "time": 1586847860000 + }, + "lastModified": { + "time": 1586847980000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.is_native", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.short_effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "is_main_series", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.is_hidden", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.slot", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "partitionsSummary", + "aspect": { + "json": { + "minPartition": { + "partition": "year=2019/month=feb", + "createdTime": 1586847860000, + "lastModifiedTime": 1586847870000 + }, + "maxPartition": { + "partition": "year=2022/month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847990000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_all.json", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json new file mode 100644 index 0000000000000..9f45583f1d3ec --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json @@ -0,0 +1,1483 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part3.csv" + }, + "name": "folder_aaa.food_csv", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_csv", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "bucket_name": "my-test-bucket" + }, + "name": "my-test-bucket" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847820000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "S3 bucket" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "UAT", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" + }, + "name": "folder_aaa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" + }, + "name": "folder_aaa.food_parquet", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.food_parquet", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "healthy", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "height", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "weight", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847840000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "number_of_partitions": "2" + }, + "name": "folder_aaa.pokemon_abilities_json", + "description": "", + "created": { + "time": 1586847860000 + }, + "lastModified": { + "time": 1586847980000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.is_native", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_changes.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "effect_entries.short_effect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "generation.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "is_main_series", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.language.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "names.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.is_hidden", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.name", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.pokemon.url", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "pokemon.slot", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "partitionsSummary", + "aspect": { + "json": { + "minPartition": { + "partition": "year=2019/month=feb", + "createdTime": 1586847860000, + "lastModifiedTime": 1586847870000 + }, + "maxPartition": { + "partition": "year=2022/month=jan", + "createdTime": 1586847980000, + "lastModifiedTime": 1586847980000 + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847990000 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "urn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577" + }, + { + "id": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "urn": "urn:li:container:86297df39321e4948dbe8b8e941de98b" + }, + { + "id": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "urn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994" + }, + { + "id": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "urn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_with_partition_autodetect_traverse_min_max.json", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json index d7a9bca716fd6..d97cedd39d77c 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json @@ -65,19 +65,19 @@ }, "fields": [ { - "fieldPath": "Sampling Date", + "fieldPath": "2", "nullable": false, "type": { "type": { - "com.linkedin.schema.DateType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "date", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "Site ID", + "fieldPath": "3", "nullable": false, "type": { "type": { @@ -89,7 +89,7 @@ "isPartOfKey": false }, { - "fieldPath": "Park ID", + "fieldPath": "Br \n(mg/L)", "nullable": false, "type": { "type": { @@ -101,7 +101,7 @@ "isPartOfKey": false }, { - "fieldPath": "Lat (\u00b0N)", + "fieldPath": "Ca \n(mg/L)", "nullable": false, "type": { "type": { @@ -113,7 +113,7 @@ "isPartOfKey": false }, { - "fieldPath": "Long (\u00b0W)", + "fieldPath": "Cl \n(mg/L)", "nullable": false, "type": { "type": { @@ -125,7 +125,7 @@ "isPartOfKey": false }, { - "fieldPath": "Water Temp (\u00b0C)", + "fieldPath": "Cond (\u00b5S/cm)", "nullable": false, "type": { "type": { @@ -137,31 +137,31 @@ "isPartOfKey": false }, { - "fieldPath": "Cond (\u00b5S/cm)", + "fieldPath": "DO (mg/L)", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "number", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "pH", + "fieldPath": "DOC [mg/L C]", "nullable": false, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "number", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "DO (mg/L)", + "fieldPath": "F \n(mg/L)", "nullable": false, "type": { "type": { @@ -173,19 +173,19 @@ "isPartOfKey": false }, { - "fieldPath": "Secchi Depth (m)", + "fieldPath": "K \n(mg/L)", "nullable": false, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "number", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "UV Absorbance, 254nm", + "fieldPath": "Lat (\u00b0N)", "nullable": false, "type": { "type": { @@ -197,7 +197,7 @@ "isPartOfKey": false }, { - "fieldPath": "DOC [mg/L C]", + "fieldPath": "Long (\u00b0W)", "nullable": false, "type": { "type": { @@ -209,7 +209,7 @@ "isPartOfKey": false }, { - "fieldPath": "SUVA, 254nm", + "fieldPath": "Mg \n(mg/L)", "nullable": false, "type": { "type": { @@ -245,31 +245,31 @@ "isPartOfKey": false }, { - "fieldPath": "PO4-P \n(mg P/L)", + "fieldPath": "Na \n(mg/L)", "nullable": false, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "number", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "TDN \n(mg N/L)", + "fieldPath": "PO4-P \n(mg P/L)", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "number", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "TDP \n(mg P/L)", + "fieldPath": "Park ID", "nullable": false, "type": { "type": { @@ -281,7 +281,7 @@ "isPartOfKey": false }, { - "fieldPath": "Cl \n(mg/L)", + "fieldPath": "SO4-S \n(mg/L)", "nullable": false, "type": { "type": { @@ -293,7 +293,7 @@ "isPartOfKey": false }, { - "fieldPath": "SO4-S \n(mg/L)", + "fieldPath": "SUVA, 254nm", "nullable": false, "type": { "type": { @@ -305,19 +305,19 @@ "isPartOfKey": false }, { - "fieldPath": "F \n(mg/L)", + "fieldPath": "Sampling Date", "nullable": false, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.DateType": {} } }, - "nativeDataType": "string", + "nativeDataType": "date", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "Br \n(mg/L)", + "fieldPath": "Secchi Depth (m)", "nullable": false, "type": { "type": { @@ -329,19 +329,19 @@ "isPartOfKey": false }, { - "fieldPath": "Na \n(mg/L)", + "fieldPath": "Site ID", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "number", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "K \n(mg/L)", + "fieldPath": "TDN \n(mg N/L)", "nullable": false, "type": { "type": { @@ -353,19 +353,19 @@ "isPartOfKey": false }, { - "fieldPath": "Ca \n(mg/L)", + "fieldPath": "TDP \n(mg P/L)", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "number", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "Mg \n(mg/L)", + "fieldPath": "UV Absorbance, 254nm", "nullable": false, "type": { "type": { @@ -377,19 +377,19 @@ "isPartOfKey": false }, { - "fieldPath": "d18O", + "fieldPath": "Water Temp (\u00b0C)", "nullable": false, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "number", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "dD", + "fieldPath": "d18O", "nullable": false, "type": { "type": { @@ -401,7 +401,7 @@ "isPartOfKey": false }, { - "fieldPath": "field29", + "fieldPath": "dD", "nullable": false, "type": { "type": { @@ -413,7 +413,7 @@ "isPartOfKey": false }, { - "fieldPath": "2", + "fieldPath": "field29", "nullable": false, "type": { "type": { @@ -425,7 +425,7 @@ "isPartOfKey": false }, { - "fieldPath": "3", + "fieldPath": "pH", "nullable": false, "type": { "type": { @@ -447,18 +447,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847610000 + "lastUpdatedTimestamp": 1586847660000 } }, "systemMetadata": { @@ -468,19 +468,13 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "container", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "instance": "test-platform-instance", - "env": "DEV", - "bucket_name": "my-test-bucket" - }, - "name": "my-test-bucket" + "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" } }, "systemMetadata": { @@ -490,8 +484,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -506,14 +500,19 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "operation", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847610000 } }, "systemMetadata": { @@ -523,15 +522,14 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "typeNames": [ - "S3 bucket" - ] + "platform": "urn:li:dataPlatform:s3", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" } }, "systemMetadata": { @@ -544,27 +542,6 @@ "entityType": "container", "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", - "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { "json": { @@ -572,9 +549,9 @@ "platform": "s3", "instance": "test-platform-instance", "env": "DEV", - "folder_abs_path": "my-test-bucket/folder_a" + "bucket_name": "my-test-bucket" }, - "name": "folder_a" + "name": "my-test-bucket" } }, "systemMetadata": { @@ -585,7 +562,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -601,7 +578,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -617,31 +594,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Folder" - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "datasetProperties", "aspect": { "json": { - "container": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv", + "number_of_files": "1", + "size_in_bytes": "172" + }, + "name": "small.csv", + "description": "", + "tags": [] } }, "systemMetadata": { @@ -652,7 +618,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -661,10 +627,6 @@ { "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - }, - { - "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", - "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" } ] } @@ -677,18 +639,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "instance": "test-platform-instance", - "env": "DEV", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "typeNames": [ + "S3 bucket" + ] } }, "systemMetadata": { @@ -698,13 +656,34 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] } }, "systemMetadata": { @@ -714,14 +693,91 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "schemaMetadata", "aspect": { "json": { + "schemaName": "small.csv", "platform": "urn:li:dataPlatform:s3", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "1st chord", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "2nd chord", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "3rd chord", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "4th chord", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Progression Quality", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + } + ] } }, "systemMetadata": { @@ -732,14 +788,18 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "containerProperties", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "customProperties": { + "platform": "s3", + "instance": "test-platform-instance", + "env": "DEV", + "folder_abs_path": "my-test-bucket/folder_a" + }, + "name": "folder_a" } }, "systemMetadata": { @@ -750,12 +810,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:c8d940d2010edd365619411b385b11e4" + "container": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" } }, "systemMetadata": { @@ -766,25 +826,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - }, - { - "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", - "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" - }, - { - "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", - "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" - } - ] + "removed": false } }, "systemMetadata": { @@ -795,7 +842,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { @@ -804,9 +851,9 @@ "platform": "s3", "instance": "test-platform-instance", "env": "DEV", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" }, - "name": "folder_aaa" + "name": "folder_aa" } }, "systemMetadata": { @@ -817,7 +864,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -833,7 +880,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -850,7 +897,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -868,12 +915,14 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { @@ -884,7 +933,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -901,10 +950,48 @@ { "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" }, { - "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", - "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" } ] } @@ -916,13 +1003,35 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + "container": "urn:li:container:c8d940d2010edd365619411b385b11e4" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { @@ -972,17 +1081,42 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "container", "aspect": { "json": { - "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro", - "number_of_files": "1", - "size_in_bytes": "619" - }, - "name": "chord_progressions_avro.avro", - "description": "", - "tags": [] + "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false } }, "systemMetadata": { @@ -1012,10 +1146,73 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", "changeType": "UPSERT", - "aspectName": "schemaMetadata", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro", + "number_of_files": "1", + "size_in_bytes": "619" + }, + "name": "chord_progressions_avro.avro", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "schemaName": "chord_progressions_avro.avro", + "platform": "urn:li:dataPlatform:s3", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv", + "number_of_files": "1", + "size_in_bytes": "604" + }, + "name": "chord_progressions_csv.csv", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "chord_progressions_csv.csv", "platform": "urn:li:dataPlatform:s3", "version": 0, "created": { @@ -1034,62 +1231,62 @@ }, "fields": [ { - "fieldPath": "[version=2.0].[type=Record].[type=long].FirstChord", + "fieldPath": "1st chord", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "FirstChord", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "[version=2.0].[type=Record].[type=long].FourthChord", + "fieldPath": "2nd chord", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "FourthChord", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "[version=2.0].[type=Record].[type=long].SecondChord", + "fieldPath": "3rd chord", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "SecondChord", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "[version=2.0].[type=Record].[type=long].ThirdChord", + "fieldPath": "4th chord", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "ThirdChord", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "[version=2.0].[type=Record].[type=string].ProgressionQuality", + "fieldPath": "Progression Quality", "nullable": false, "type": { "type": { "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "ProgressionQuality", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false } @@ -1102,44 +1299,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847620000 - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", @@ -1179,52 +1338,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:s3", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv", - "number_of_files": "1", - "size_in_bytes": "604" - }, - "name": "chord_progressions_csv.csv", - "description": "", - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "chord_progressions_csv.csv", + "schemaName": "chord_progressions_avro.avro", "platform": "urn:li:dataPlatform:s3", "version": 0, "created": { @@ -1243,62 +1362,62 @@ }, "fields": [ { - "fieldPath": "1st chord", + "fieldPath": "[version=2.0].[type=Record].[type=long].FirstChord", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "FirstChord", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "2nd chord", + "fieldPath": "[version=2.0].[type=Record].[type=long].FourthChord", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "FourthChord", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "3rd chord", + "fieldPath": "[version=2.0].[type=Record].[type=long].SecondChord", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "SecondChord", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "4th chord", + "fieldPath": "[version=2.0].[type=Record].[type=long].ThirdChord", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "ThirdChord", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "Progression Quality", + "fieldPath": "[version=2.0].[type=Record].[type=string].ProgressionQuality", "nullable": false, "type": { "type": { "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "string", + "nativeDataType": "ProgressionQuality", "recursive": false, "isPartOfKey": false } @@ -1312,35 +1431,19 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "containerProperties", "aspect": { "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "customProperties": { + "platform": "s3", + "instance": "test-platform-instance", + "env": "DEV", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa/folder_aaa" }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847630000 - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + "name": "folder_aaa" } }, "systemMetadata": { @@ -1351,33 +1454,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - }, - { - "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", - "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" - }, - { - "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", - "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" - }, - { - "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", - "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" - }, - { - "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", - "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" - } - ] + "platform": "urn:li:dataPlatform:s3", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" } }, "systemMetadata": { @@ -1409,23 +1492,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:s3", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", @@ -1498,18 +1564,55 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + }, + { + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" + }, + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" + }, + { + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + }, + { + "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847640000 + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { @@ -1520,7 +1623,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1573,19 +1676,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "operation", "aspect": { "json": { - "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet", - "number_of_files": "1", - "size_in_bytes": "4206" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "name": "food_parquet.parquet", - "description": "", - "tags": [] + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847640000 } }, "systemMetadata": { @@ -1596,7 +1698,55 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -1612,89 +1762,46 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", "changeType": "UPSERT", - "aspectName": "schemaMetadata", + "aspectName": "subTypes", "aspect": { "json": { - "schemaName": "food_parquet.parquet", - "platform": "urn:li:dataPlatform:s3", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "color", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false - }, + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ { - "fieldPath": "healthy", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.BooleanType": {} - } - }, - "nativeDataType": "bool", - "recursive": false, - "isPartOfKey": false + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" }, { - "fieldPath": "height", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "int64", - "recursive": false, - "isPartOfKey": false + "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" }, - { - "fieldPath": "name", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false + { + "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", + "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" }, { - "fieldPath": "weight", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.NumberType": {} - } - }, - "nativeDataType": "int64", - "recursive": false, - "isPartOfKey": false + "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", + "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" } ] } @@ -1707,18 +1814,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847650000 + "lastUpdatedTimestamp": 1586847670000 } }, "systemMetadata": { @@ -1729,7 +1836,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1745,33 +1852,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - }, - { - "id": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", - "urn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689" - }, - { - "id": "urn:li:container:c8d940d2010edd365619411b385b11e4", - "urn": "urn:li:container:c8d940d2010edd365619411b385b11e4" - }, - { - "id": "urn:li:container:b0037296cdd497e3137aa0628b8687bc", - "urn": "urn:li:container:b0037296cdd497e3137aa0628b8687bc" - }, - { - "id": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9", - "urn": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" - } - ] + "removed": false } }, "systemMetadata": { @@ -1782,7 +1868,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -1799,17 +1885,34 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:s3", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv", + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet", "number_of_files": "1", - "size_in_bytes": "172" + "size_in_bytes": "4206" }, - "name": "small.csv", + "name": "food_parquet.parquet", "description": "", "tags": [] } @@ -1822,12 +1925,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "small.csv", + "schemaName": "food_parquet.parquet", "platform": "urn:li:dataPlatform:s3", "version": 0, "created": { @@ -1846,62 +1949,62 @@ }, "fields": [ { - "fieldPath": "1st chord", + "fieldPath": "color", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "2nd chord", + "fieldPath": "healthy", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.BooleanType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "bool", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "3rd chord", + "fieldPath": "height", "nullable": false, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "int64", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "4th chord", + "fieldPath": "name", "nullable": false, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "Progression Quality", + "fieldPath": "weight", "nullable": false, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "int64", "recursive": false, "isPartOfKey": false } @@ -1916,34 +2019,19 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "datasetProperties", "aspect": { "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "customProperties": { + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv", + "number_of_files": "1", + "size_in_bytes": "34056" }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847660000 - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" + "name": "wa_fn_usec_hr_employee_attrition_csv.csv", + "description": "", + "tags": [] } }, "systemMetadata": { @@ -1954,7 +2042,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -1989,29 +2077,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "datasetProperties", - "aspect": { - "json": { - "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv", - "number_of_files": "1", - "size_in_bytes": "34056" - }, - "name": "wa_fn_usec_hr_employee_attrition_csv.csv", - "description": "", - "tags": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", @@ -2420,62 +2485,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "operation", - "aspect": { - "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847670000 - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:s3", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:s3,test-platform-instance)" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -2512,28 +2522,18 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_files.json", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "operation", "aspect": { "json": { - "removed": false + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847650000 } }, "systemMetadata": { @@ -2544,12 +2544,12 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:de5780654849d6a18b66df2f9cb8e8d9" } }, "systemMetadata": { @@ -2560,7 +2560,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2592,7 +2592,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2608,7 +2608,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json index 9c41bbdc80c49..5d45a2a5f696e 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -111,7 +112,45 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "PROD", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -133,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -145,8 +185,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1586847620000 @@ -154,32 +194,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "s3", - "env": "PROD", - "bucket_name": "my-test-bucket" - }, - "name": "my-test-bucket" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -189,54 +210,47 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:s3" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "S3 bucket" + "Folder" ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "containerProperties", "aspect": { "json": { - "path": [] + "customProperties": { + "platform": "s3", + "env": "PROD", + "bucket_name": "my-test-bucket" + }, + "name": "my-test-bucket" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -256,7 +270,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -271,7 +286,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -286,39 +302,42 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -338,89 +357,90 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "PROD", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "container": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "path": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + "typeNames": [ + "S3 bucket" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -444,7 +464,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -464,37 +485,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -511,22 +535,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -554,7 +580,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -569,39 +596,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", - "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" - }, - { - "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", - "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" - }, - { - "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", - "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" - }, - { - "id": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", - "urn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" - } - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -623,7 +635,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -716,65 +729,102 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "browsePathsV2", "aspect": { "json": { - "tags": [ + "path": [ { - "tag": "urn:li:tag:baz:bob" + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" }, { - "tag": "urn:li:tag:foo:bar" + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + }, + { + "id": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "urn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:baz:bob", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "baz:bob" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "globalTags", "aspect": { "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" - }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847630000 + "tags": [ + { + "tag": "urn:li:tag:baz:bob" + }, + { + "tag": "urn:li:tag:foo:bar" + } + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "operation", "aspect": { "json": { - "container": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -806,22 +856,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -836,22 +888,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:baz:bob", - "changeType": "UPSERT", - "aspectName": "tagKey", - "aspect": { - "json": { - "name": "baz:bob" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } }, { @@ -866,7 +904,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_spec_for_files.json" + "runId": "multiple_spec_for_files.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json index 985140f774ab4..90c3e0b3850b7 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -111,7 +112,45 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "PROD", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -133,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -145,8 +185,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1586847620000 @@ -154,32 +194,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "s3", - "env": "PROD", - "bucket_name": "my-test-bucket" - }, - "name": "my-test-bucket" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -189,54 +210,47 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:s3" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "S3 bucket" + "Folder" ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "containerProperties", "aspect": { "json": { - "path": [] + "customProperties": { + "platform": "s3", + "env": "PROD", + "bucket_name": "my-test-bucket" + }, + "name": "my-test-bucket" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -256,7 +270,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -271,7 +286,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -286,39 +302,42 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -338,89 +357,90 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "PROD", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "container": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "path": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + "typeNames": [ + "S3 bucket" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -444,7 +464,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -464,37 +485,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -511,22 +535,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -554,7 +580,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -569,39 +596,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", - "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" - }, - { - "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", - "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" - }, - { - "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", - "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" - }, - { - "id": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", - "urn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" - } - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -623,7 +635,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -716,100 +729,123 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket-2/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "browsePathsV2", "aspect": { "json": { - "tags": [ + "path": [ { - "tag": "urn:li:tag:baz:bob" + "id": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "urn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" }, { - "tag": "urn:li:tag:foo:bar" + "id": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "urn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + }, + { + "id": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "urn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + }, + { + "id": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "urn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket-2/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "containerProperties", "aspect": { "json": { - "timestampMillis": 1615443388097, - "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "customProperties": { + "platform": "s3", + "env": "PROD", + "bucket_name": "my-test-bucket-2" }, - "operationType": "UPDATE", - "lastUpdatedTimestamp": 1586847630000 + "name": "my-test-bucket-2" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket-2/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "globalTags", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "PROD", - "bucket_name": "my-test-bucket-2" - }, - "name": "my-test-bucket-2" + "tags": [ + { + "tag": "urn:li:tag:baz:bob" + }, + { + "tag": "urn:li:tag:foo:bar" + } + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket-2/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "operation", "aspect": { "json": { - "removed": false + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "operationType": "UPDATE", + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -826,22 +862,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "path": [] + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -861,7 +899,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -876,7 +915,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -891,39 +931,42 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:f6d8484efac8152d10620c6c0699d02d", + "entityUrn": "urn:li:container:680e54d5e3a7705caa1d99893fab4924", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "path": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:f6d8484efac8152d10620c6c0699d02d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:680e54d5e3a7705caa1d99893fab4924" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -943,7 +986,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -963,7 +1007,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -978,7 +1023,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -993,39 +1039,42 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0", + "entityType": "tag", + "entityUrn": "urn:li:tag:baz:bob", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "tagKey", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "name": "baz:bob" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:f6d8484efac8152d10620c6c0699d02d" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1049,7 +1098,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1069,37 +1119,72 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f6d8484efac8152d10620c6c0699d02d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:680e54d5e3a7705caa1d99893fab4924" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f6d8484efac8152d10620c6c0699d02d" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ce2eca2107ef4c0b47a8f4a65eff971c", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ce2eca2107ef4c0b47a8f4a65eff971c", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1116,22 +1201,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:ce2eca2107ef4c0b47a8f4a65eff971c", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:6e8c28494477b4a90cf5fd395217bae0" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1159,7 +1246,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1174,7 +1262,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1189,7 +1278,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1221,37 +1311,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" - } -}, -{ - "entityType": "tag", - "entityUrn": "urn:li:tag:baz:bob", - "changeType": "UPSERT", - "aspectName": "tagKey", - "aspect": { - "json": { - "name": "baz:bob" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } }, { @@ -1266,7 +1327,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "multiple_specs_of_different_buckets.json" + "runId": "multiple_specs_of_different_buckets.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json index 5d87d423a6a67..5fb691c1ee457 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json @@ -18,7 +18,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -111,7 +112,45 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "s3", + "env": "PROD", + "folder_abs_path": "my-test-bucket/folder_a/folder_aa" + }, + "name": "folder_aa" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -133,7 +172,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -145,8 +185,8 @@ "json": { "timestampMillis": 1615443388097, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "operationType": "UPDATE", "lastUpdatedTimestamp": 1586847620000 @@ -154,32 +194,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "s3", - "env": "PROD", - "bucket_name": "my-test-bucket" - }, - "name": "my-test-bucket" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -189,54 +210,47 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:s3" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "single_file.json" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "S3 bucket" + "Folder" ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "containerProperties", "aspect": { "json": { - "path": [] + "customProperties": { + "platform": "s3", + "env": "PROD", + "bucket_name": "my-test-bucket" + }, + "name": "my-test-bucket" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -256,7 +270,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -271,7 +286,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -286,39 +302,42 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" + "typeNames": [ + "Folder" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -338,89 +357,90 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "customProperties": { - "platform": "s3", - "env": "PROD", - "folder_abs_path": "my-test-bucket/folder_a/folder_aa" - }, - "name": "folder_aa" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "container": "urn:li:container:2151647ff17bde0f948909d19fa91b9b" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "Folder" - ] + "path": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", - "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "subTypes", "aspect": { "json": { - "container": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c" + "typeNames": [ + "S3 bucket" + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -444,7 +464,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -464,37 +485,40 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "container", "aspect": { "json": { - "removed": false + "container": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "status", "aspect": { "json": { - "platform": "urn:li:dataPlatform:s3" + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -511,22 +535,24 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { "entityType": "container", "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "container": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15" + "platform": "urn:li:dataPlatform:s3" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -554,7 +580,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -569,7 +596,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -584,7 +612,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -616,7 +645,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -631,7 +661,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } }, { @@ -646,7 +677,8 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "single_file.json" + "runId": "single_file.json", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition.json index c06e411005399..db3eaadf30040 100644 --- a/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition.json @@ -2,6 +2,7 @@ "type": "s3", "config": { "env": "UAT", + "sort_schema_fields": true, "path_specs": [{ "include": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/{table}/*.*" }], diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_update_schema_with_partition_autodetect.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_update_schema_with_partition_autodetect.json new file mode 100644 index 0000000000000..2ed1a4ae9ccd3 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_update_schema_with_partition_autodetect.json @@ -0,0 +1,23 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_specs": [{ + "include": "s3://my-test-bucket/folder_a/folder_aa/{dept}/{table}/", + "sample_files": true, + "autodetect_partitions": true, + "table_name": "{dept}.{table}", + "exclude":[ + "**/folder_aaaa/**" + ] + }], + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_all.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_all.json new file mode 100644 index 0000000000000..b6aa75822612a --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_all.json @@ -0,0 +1,24 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_specs": [{ + "include": "s3://my-test-bucket/folder_a/folder_aa/{dept}/{table}/", + "sample_files": true, + "autodetect_partitions": true, + "traversal_method": "ALL", + "table_name": "{dept}.{table}", + "exclude":[ + "**/folder_aaaa/**" + ] + }], + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_min_max.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_min_max.json new file mode 100644 index 0000000000000..ecf4fab1f4b07 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_min_max.json @@ -0,0 +1,24 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_specs": [{ + "include": "s3://my-test-bucket/folder_a/folder_aa/{dept}/{table}/", + "sample_files": true, + "autodetect_partitions": true, + "traversal_method": "MIN_MAX", + "table_name": "{dept}.{table}", + "exclude":[ + "**/folder_aaaa/**" + ] + }], + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv b/metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv index e69de29bb2d1d..c757de4cc1d77 100644 --- a/metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv +++ b/metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part3.csv @@ -0,0 +1,5 @@ +name,weight,height,color +apple,50,4,red +cookie,23,4,brown +lasagna,49,4,red +pasta,72,4,yellow diff --git a/metadata-ingestion/tests/integration/s3/test_s3.py b/metadata-ingestion/tests/integration/s3/test_s3.py index d255463444b18..4137c6c5c399e 100644 --- a/metadata-ingestion/tests/integration/s3/test_s3.py +++ b/metadata-ingestion/tests/integration/s3/test_s3.py @@ -15,6 +15,48 @@ FROZEN_TIME = "2020-04-14 07:00:00" +FILE_LIST_FOR_VALIDATION = [ + "folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv", + "folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro", + "folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv", + "folder_a/folder_aa/folder_aaa/countries_json.json", + "folder_a/folder_aa/folder_aaa/food_parquet.parquet", + "folder_a/folder_aa/folder_aaa/small.csv", + "folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2019/month=feb/part1.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2019/month=feb/part2.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2019/month=jan/part1.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2019/month=jan/part2.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2020/month=feb/part1.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2020/month=feb/part2.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2020/month=march/part1.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2020/month=march/part2.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2021/month=april/part1.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2021/month=april/part2.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2021/month=march/part1.json", + "folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2021/month=march/part2.json", + "folder_a/folder_aa/folder_aaa/food_csv/part1.csv", + "folder_a/folder_aa/folder_aaa/food_csv/part2.csv", + "folder_a/folder_aa/folder_aaa/food_csv/part3.csv", + "folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet", + "folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet", + "folder_a/folder_aa/folder_aaa/no_extension/small", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=feb/part1.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=feb/part2.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=jan/part1.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=jan/part2.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2020/month=feb/part1.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2020/month=feb/part2.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2020/month=march/part1.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2020/month=march/part2.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2021/month=april/part1.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2021/month=april/part2.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2021/month=march/part1.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2021/month=march/part2.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json", + "folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/_temporary/dummy.json", +] + @pytest.fixture(scope="module", autouse=True) def bucket_names(): @@ -60,10 +102,13 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names): current_time_sec = datetime.strptime( FROZEN_TIME, "%Y-%m-%d %H:%M:%S" ).timestamp() + file_list = [] for root, _dirs, files in os.walk(test_resources_dir): + _dirs.sort() for file in sorted(files): full_path = os.path.join(root, file) rel_path = os.path.relpath(full_path, test_resources_dir) + file_list.append(rel_path) bkt.upload_file(full_path, rel_path) s3_client.put_object_tagging( Bucket=bucket_name, @@ -77,6 +122,9 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names): ) current_time_sec += 10 key.last_modified = datetime.fromtimestamp(current_time_sec) + + # This is used to make sure the list of files are the same in the test as locally + assert file_list == FILE_LIST_FOR_VALIDATION yield diff --git a/metadata-ingestion/tests/unit/s3/test_s3_source.py b/metadata-ingestion/tests/unit/s3/test_s3_source.py index 2eb386e39b0e5..f826cf0179e22 100644 --- a/metadata-ingestion/tests/unit/s3/test_s3_source.py +++ b/metadata-ingestion/tests/unit/s3/test_s3_source.py @@ -1,4 +1,6 @@ -from typing import List +from typing import List, Tuple + +import pytest from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.workunit import MetadataWorkUnit @@ -70,6 +72,75 @@ def test_path_spec(): assert path_spec.allowed(path) +def test_path_spec_with_double_star_ending(): + path_spec = PathSpec( + include="s3://my-bucket/{table}/**", + default_extension="csv", + allow_double_stars=True, + ) + path = "s3://my-bucket/my-folder/year=2022/month=10/day=11/my_csv.csv" + assert path_spec.allowed(path) + vars = path_spec.get_named_vars(path) + assert vars + assert vars["table"] == "my-folder" + + +@pytest.mark.parametrize( + "path_spec,path, expected", + [ + pytest.param( + "s3://my-bucket/{table}/**", + "s3://my-bucket/my-folder/year=2022/month=10/day=11/my_csv", + [("year", "2022"), ("month", "10"), ("day", "11")], + id="autodetect_partitions", + ), + pytest.param( + "s3://my-bucket/{table}/{partition_key[0]}={partition_value[0]}/{partition_key[1]}={partition_value[1]}/{partition_key[2]}={partition_value[2]}/*.csv", + "s3://my-bucket/my-folder/year=2022/month=10/day=11/my_csv.csv", + [("year", "2022"), ("month", "10"), ("day", "11")], + id="partition_key and value set", + ), + pytest.param( + "s3://my-bucket/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/{partition_key[2]}={partition[2]}/*.csv", + "s3://my-bucket/my-folder/year=2022/month=10/day=11/my_csv.csv", + [("year", "2022"), ("month", "10"), ("day", "11")], + id="partition_key and partition set", + ), + pytest.param( + "s3://my-bucket/{table}/{year}/{month}/{day}/*.csv", + "s3://my-bucket/my-folder/2022/10/11/my_csv.csv", + [("year", "2022"), ("month", "10"), ("day", "11")], + id="named partition keys", + ), + pytest.param( + "s3://my-bucket/{table}/{part[0]}/{part[1]}/{part[2]}/*.csv", + "s3://my-bucket/my-folder/2022/10/11/my_csv.csv", + [("part_0", "2022"), ("part_1", "10"), ("part_2", "11")], + id="indexed partition keys", + ), + pytest.param( + "s3://my-bucket/{table}/**", + "s3://my-bucket/my-folder/2022/10/11/my_csv.csv", + [("partition_0", "2022"), ("partition_1", "10"), ("partition_2", "11")], + id="partition autodetect with partition values only", + ), + pytest.param( + "s3://my-bucket/{table}/**", + "s3://my-bucket/my-folder/my_csv.csv", + None, + id="partition autodetect with non partitioned path", + ), + ], +) +def test_path_spec_partition_detection( + path_spec: str, path: str, expected: List[Tuple[str, str]] +) -> None: + ps = PathSpec(include=path_spec, default_extension="csv", allow_double_stars=True) + assert ps.allowed(path) + partitions = ps.get_partition_from_path(path) + assert partitions == expected + + def test_path_spec_dir_allowed(): path_spec = PathSpec( include="s3://my-bucket/my-folder/year=*/month=*/day=*/*.csv",