diff --git a/dbt_project.yml b/dbt_project.yml index f957879..f213b50 100755 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,45 +1,29 @@ -# This file is necessary to install dbt-utils with dbt deps -# the content will be overwritten by the transform function - -# Name your package! Package names should contain only lowercase characters -# and underscores. A good package name should reflect your organization's -# name or the intended use of these models -name: "airbyte_utils" -version: "1.0" +name: airbyte_utils +version: '1.0' config-version: 2 - -# This setting configures which "profile" dbt uses for this project. Profiles contain -# database connection information, and should be configured in the ~/.dbt/profiles.yml file -profile: "normalize" - -# These configurations specify where dbt should look for different types of files. -# The `model-paths` config, for example, states that source models can be found -# in the "models/" directory. You probably won't need to change these! -model-paths: ["models"] -docs-paths: ["docs"] -analysis-paths: ["analysis"] -test-paths: ["tests"] -seed-paths: ["data"] -macro-paths: ["macros"] - -target-path: "../build" # directory which will store compiled SQL files -log-path: "../logs" # directory which will store DBT logs -packages-install-path: "../dbt" # directory which will store external DBT dependencies - -clean-targets: # directories to be removed by `dbt clean` - - "build" - - "dbt_modules" - +profile: normalize +model-paths: +- models +docs-paths: +- docs +analysis-paths: +- analysis +test-paths: +- tests +seed-paths: +- data +macro-paths: +- macros +target-path: ../build +log-path: ../logs +packages-install-path: ../dbt +clean-targets: +- build +- dbt_modules quoting: database: true - # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) - # all schemas should be unquoted schema: false identifier: true - -# You can define configurations for models in the `model-paths` directory here. -# Using these configurations, you can enable or disable models, change how they -# are materialized, and more! models: airbyte_utils: +materialized: table @@ -57,7 +41,151 @@ models: airbyte_views: +tags: airbyte_internal_views +materialized: view - dispatch: - - macro_namespace: dbt_utils - search_order: ["airbyte_utils", "dbt_utils"] +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils +vars: + json_column: _airbyte_data + models_to_source: + admin_tags_ab1: unibag._airbyte_raw_admin_tags + admin_tags_ab2: unibag._airbyte_raw_admin_tags + admin_tags_ab3: unibag._airbyte_raw_admin_tags + admin_tags: unibag._airbyte_raw_admin_tags + cash_flows_ab1: unibag._airbyte_raw_cash_flows + cash_flows_ab2: unibag._airbyte_raw_cash_flows + cash_flows_ab3: unibag._airbyte_raw_cash_flows + cash_flows: unibag._airbyte_raw_cash_flows + cities_ab1: unibag._airbyte_raw_cities + cities_ab2: unibag._airbyte_raw_cities + cities_ab3: unibag._airbyte_raw_cities + cities: unibag._airbyte_raw_cities + customers_ab1: unibag._airbyte_raw_customers + customers_ab2: unibag._airbyte_raw_customers + customers_ab3: unibag._airbyte_raw_customers + customers: unibag._airbyte_raw_customers + districts_ab1: unibag._airbyte_raw_districts + districts_ab2: unibag._airbyte_raw_districts + districts_ab3: unibag._airbyte_raw_districts + districts: unibag._airbyte_raw_districts + event_rewards_ab1: unibag._airbyte_raw_event_rewards + event_rewards_ab2: unibag._airbyte_raw_event_rewards + event_rewards_ab3: unibag._airbyte_raw_event_rewards + event_rewards: unibag._airbyte_raw_event_rewards + events_ab1: unibag._airbyte_raw_events + events_ab2: unibag._airbyte_raw_events + events_ab3: unibag._airbyte_raw_events + events: unibag._airbyte_raw_events + inventories_ab1: unibag._airbyte_raw_inventories + inventories_ab2: unibag._airbyte_raw_inventories + inventories_ab3: unibag._airbyte_raw_inventories + inventories: unibag._airbyte_raw_inventories + membership_ab1: unibag._airbyte_raw_membership + membership_ab2: unibag._airbyte_raw_membership + membership_ab3: unibag._airbyte_raw_membership + membership: unibag._airbyte_raw_membership + membership_histories_ab1: unibag._airbyte_raw_membership_histories + membership_histories_ab2: unibag._airbyte_raw_membership_histories + membership_histories_ab3: unibag._airbyte_raw_membership_histories + membership_histories: unibag._airbyte_raw_membership_histories + membership_orders_ab1: unibag._airbyte_raw_membership_orders + membership_orders_ab2: unibag._airbyte_raw_membership_orders + membership_orders_ab3: unibag._airbyte_raw_membership_orders + membership_orders: unibag._airbyte_raw_membership_orders + order_deliveries_ab1: unibag._airbyte_raw_order_deliveries + order_deliveries_ab2: unibag._airbyte_raw_order_deliveries + order_deliveries_ab3: unibag._airbyte_raw_order_deliveries + order_deliveries: unibag._airbyte_raw_order_deliveries + order_histories_ab1: unibag._airbyte_raw_order_histories + order_histories_ab2: unibag._airbyte_raw_order_histories + order_histories_ab3: unibag._airbyte_raw_order_histories + order_histories: unibag._airbyte_raw_order_histories + order_items_ab1: unibag._airbyte_raw_order_items + order_items_ab2: unibag._airbyte_raw_order_items + order_items_ab3: unibag._airbyte_raw_order_items + order_items: unibag._airbyte_raw_order_items + orders_ab1: unibag._airbyte_raw_orders + orders_ab2: unibag._airbyte_raw_orders + orders_ab3: unibag._airbyte_raw_orders + orders: unibag._airbyte_raw_orders + product_categories_ab1: unibag._airbyte_raw_product_categories + product_categories_ab2: unibag._airbyte_raw_product_categories + product_categories_ab3: unibag._airbyte_raw_product_categories + product_categories: unibag._airbyte_raw_product_categories + product_sku_ab1: unibag._airbyte_raw_product_sku + product_sku_ab2: unibag._airbyte_raw_product_sku + product_sku_ab3: unibag._airbyte_raw_product_sku + product_sku: unibag._airbyte_raw_product_sku + product_sku_versions_ab1: unibag._airbyte_raw_product_sku_versions + product_sku_versions_ab2: unibag._airbyte_raw_product_sku_versions + product_sku_versions_ab3: unibag._airbyte_raw_product_sku_versions + product_sku_versions: unibag._airbyte_raw_product_sku_versions + product_sub_categories_ab1: unibag._airbyte_raw_product_sub_categories + product_sub_categories_ab2: unibag._airbyte_raw_product_sub_categories + product_sub_categories_ab3: unibag._airbyte_raw_product_sub_categories + product_sub_categories: unibag._airbyte_raw_product_sub_categories + products_ab1: unibag._airbyte_raw_products + products_ab2: unibag._airbyte_raw_products + products_ab3: unibag._airbyte_raw_products + products: unibag._airbyte_raw_products + promotion_orders_ab1: unibag._airbyte_raw_promotion_orders + promotion_orders_ab2: unibag._airbyte_raw_promotion_orders + promotion_orders_ab3: unibag._airbyte_raw_promotion_orders + promotion_orders: unibag._airbyte_raw_promotion_orders + promotions_ab1: unibag._airbyte_raw_promotions + promotions_ab2: unibag._airbyte_raw_promotions + promotions_ab3: unibag._airbyte_raw_promotions + promotions: unibag._airbyte_raw_promotions + referrals_ab1: unibag._airbyte_raw_referrals + referrals_ab2: unibag._airbyte_raw_referrals + referrals_ab3: unibag._airbyte_raw_referrals + referrals: unibag._airbyte_raw_referrals + social_post_views_ab1: unibag._airbyte_raw_social_post_views + social_post_views_ab2: unibag._airbyte_raw_social_post_views + social_post_views_ab3: unibag._airbyte_raw_social_post_views + social_post_views: unibag._airbyte_raw_social_post_views + social_posts_ab1: unibag._airbyte_raw_social_posts + social_posts_ab2: unibag._airbyte_raw_social_posts + social_posts_ab3: unibag._airbyte_raw_social_posts + social_posts: unibag._airbyte_raw_social_posts + suppliers_ab1: unibag._airbyte_raw_suppliers + suppliers_ab2: unibag._airbyte_raw_suppliers + suppliers_ab3: unibag._airbyte_raw_suppliers + suppliers: unibag._airbyte_raw_suppliers + team_activities_ab1: unibag._airbyte_raw_team_activities + team_activities_ab2: unibag._airbyte_raw_team_activities + team_activities_ab3: unibag._airbyte_raw_team_activities + team_activities: unibag._airbyte_raw_team_activities + team_bonus_ab1: unibag._airbyte_raw_team_bonus + team_bonus_ab2: unibag._airbyte_raw_team_bonus + team_bonus_ab3: unibag._airbyte_raw_team_bonus + team_bonus: unibag._airbyte_raw_team_bonus + team_members_ab1: unibag._airbyte_raw_team_members + team_members_ab2: unibag._airbyte_raw_team_members + team_members_ab3: unibag._airbyte_raw_team_members + team_members: unibag._airbyte_raw_team_members + teams_ab1: unibag._airbyte_raw_teams + teams_ab2: unibag._airbyte_raw_teams + teams_ab3: unibag._airbyte_raw_teams + teams: unibag._airbyte_raw_teams + user_identifications_ab1: unibag._airbyte_raw_user_identifications + user_identifications_ab2: unibag._airbyte_raw_user_identifications + user_identifications_ab3: unibag._airbyte_raw_user_identifications + user_identifications: unibag._airbyte_raw_user_identifications + users_ab1: unibag._airbyte_raw_users + users_ab2: unibag._airbyte_raw_users + users_ab3: unibag._airbyte_raw_users + users: unibag._airbyte_raw_users + voucher_bonus_ab1: unibag._airbyte_raw_voucher_bonus + voucher_bonus_ab2: unibag._airbyte_raw_voucher_bonus + voucher_bonus_ab3: unibag._airbyte_raw_voucher_bonus + voucher_bonus: unibag._airbyte_raw_voucher_bonus + vouchers_ab1: unibag._airbyte_raw_vouchers + vouchers_ab2: unibag._airbyte_raw_vouchers + vouchers_ab3: unibag._airbyte_raw_vouchers + vouchers: unibag._airbyte_raw_vouchers + wards_ab1: unibag._airbyte_raw_wards + wards_ab2: unibag._airbyte_raw_wards + wards_ab3: unibag._airbyte_raw_wards + wards: unibag._airbyte_raw_wards diff --git a/destination_catalog.json b/destination_catalog.json index 0118dcb..016b773 100644 --- a/destination_catalog.json +++ b/destination_catalog.json @@ -1 +1 @@ -{"streams":[{"stream":{"name":"admin-tags","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"active":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"cash-flows","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"hash":{"type":"string"},"user":{"type":"string"},"value":{"type":"number"},"action":{"type":"string"},"options":{"type":"string"},"category":{"type":"string"},"targetId":{"type":"string"},"createdAt":{"type":"string"},"isAudited":{"type":"boolean"},"updatedAt":{"type":"string"},"newBalance":{"type":"number"},"oldBalance":{"type":"number"},"targetType":{"type":"string"},"isProcessed":{"type":"boolean"},"processStatus":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"cities","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"order":{"type":"number"},"tncId":{"type":"number"},"region":{"type":"string"},"source":{"type":"string"},"tempId":{"type":"number"},"tncCode":{"type":"string"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"customers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"user":{"type":"string"},"phone":{"type":"string"},"source":{"type":"string"},"location":{"type":"array"},"createdAt":{"type":"string"},"isDeleted":{"type":"boolean"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"districts","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"city":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"tncId":{"type":"number"},"cityId":{"type":"number"},"source":{"type":"string"},"tncCode":{"type":"string"},"gidoCode":{"type":"number"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"event-rewards","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"cash":{"type":"number"},"type":{"type":"string"},"user":{"type":"string"},"event":{"type":"string"},"status":{"type":"string"},"options":{"type":"string"},"createdAt":{"type":"string"},"updatedat":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"events","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"desc":{"type":"string"},"icon":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"cover":{"type":"string"},"endAt":{"type":"string"},"active":{"type":"boolean"},"notice":{"type":"string"},"article":{"type":"string"},"options":{"type":"string"},"startAt":{"type":"string"},"segments":{"type":"string"},"createdAt":{"type":"string"},"suppliers":{"type":"array"},"updatedAt":{"type":"string"},"information":{"type":"array"},"displayEndAt":{"type":"string"},"isApplyForAll":{"type":"boolean"},"displayStartAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"inventories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"active":{"type":"boolean"},"contact":{"type":"string"},"partner":{"type":"string"},"checksum":{"type":"string"},"location":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"minimumValue":{"type":"number"},"searchString":{"type":"string"},"paymentMethods":{"type":"string"},"canIssueInvoice":{"type":"boolean"},"deliveryMethods":{"type":"array"},"canAutoSendEmail":{"type":"boolean"},"invoiceDeliveryMethod":{"type":"string"},"doesSupportSellyExpress":{"type":"boolean"},"priorityDeliveryServiceCodes":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"color":{"type":"string"},"guide":{"type":"string"},"level":{"type":"number"},"sales":{"type":"number"},"active":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"transaction":{"type":"number"},"bonusPercent":{"type":"number"},"transactionMinValue":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership-histories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"new":{"type":"string"},"old":{"type":"string"},"type":{"type":"string"},"user":{"type":"string"},"createdAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership-orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"status":{"type":"string"},"percent":{"type":"number"},"targetID":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"membershipName":{"type":"string"},"membershipLevel":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-deliveries","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"weight":{"type":"string"},"service":{"type":"string"},"customer":{"type":"string"},"delivery":{"type":"string"},"location":{"type":"string"},"createdAt":{"type":"string"},"promotion":{"type":"string"},"updatedAt":{"type":"string"},"trackingCode":{"type":"string"},"trackingTime":{"type":"string"},"isAdminChange":{"type":"boolean"},"trackingOrderCode":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-histories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"order":{"type":"string"},"status":{"type":"string"},"createdAt":{"type":"string"},"timeStamp":{"type":"number"},"updatedAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-items","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"date":{"type":"string"},"user":{"type":"string"},"order":{"type":"string"},"price":{"type":"string"},"status":{"type":"string"},"product":{"type":"string"},"customer":{"type":"string"},"quantity":{"type":"number"},"createdAt":{"type":"string"},"totalSell":{"type":"number"},"updatedAt":{"type":"string"},"cashbackAt":{"type":"string"},"totalPrice":{"type":"number"},"deliveredAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"date":{"type":"string"},"hour":{"type":"number"},"note":{"type":"string"},"skus":{"type":"array"},"tags":{"type":"array"},"user":{"type":"string"},"price":{"type":"string"},"banned":{"type":"boolean"},"reason":{"type":"string"},"source":{"type":"string"},"status":{"type":"string"},"update":{"type":"string"},"payment":{"type":"string"},"remarks":{"type":"string"},"customer":{"type":"string"},"delivery":{"type":"string"},"pickupAt":{"type":"string"},"products":{"type":"array"},"supplier":{"type":"string"},"approveAt":{"type":"string"},"createdAt":{"type":"string"},"inventory":{"type":"string"},"isDeleted":{"type":"boolean"},"promotion":{"type":"string"},"requestId":{"type":"string"},"sendEmail":{"type":"string"},"totalItem":{"type":"number"},"updatedAt":{"type":"string"},"userAgent":{"type":"string"},"wholesale":{"type":"boolean"},"approvedAt":{"type":"string"},"cashbackAt":{"type":"string"},"codeOsiris":{"type":"string"},"isPreorder":{"type":"boolean"},"membership":{"type":"string"},"rejectedAt":{"type":"string"},"deliveredAt":{"type":"string"},"deliveringAt":{"type":"string"},"deliveryCode":{"type":"string"},"searchString":{"type":"string"},"staffApprove":{"type":"string"},"trackingCode":{"type":"string"},"processStatus":{"type":"string"},"hookTimeLastAt":{"type":"string"},"trackingCodeURL":{"type":"string"},"approvePaymentAt":{"type":"string"},"isChangeDelivery":{"type":"boolean"},"trackingOrderCode":{"type":"string"},"estimateCashbackAt":{"type":"string"},"isWaitingCancelled":{"type":"boolean"},"fromNewActiveSeller":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-categories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"icon":{"type":"string"},"name":{"type":"string"},"color":{"type":"string"},"order":{"type":"number"},"active":{"type":"boolean"},"covers":{"type":"array"},"featured":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"totalProduct":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sku","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"brand":{"type":"string"},"cover":{"type":"string"},"price":{"type":"string"},"active":{"type":"boolean"},"author":{"type":"string"},"manual":{"type":"boolean"},"source":{"type":"string"},"display":{"type":"boolean"},"picture":{"type":"string"},"product":{"type":"string"},"version":{"type":"number"},"groupSku":{"type":"string"},"unitCode":{"type":"string"},"createdAt":{"type":"string"},"restockAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"properties":{"type":"array"},"canPreorder":{"type":"boolean"},"supplierSku":{"type":"string"},"isOutOfStock":{"type":"boolean"},"pricePercent":{"type":"string"},"searchString":{"type":"string"},"updateReason":{"type":"string"},"updateActiveId":{"type":"string"},"displayInventory":{"type":"boolean"},"showRemainingQuantity":{"type":"boolean"},"quantity_aibyte_transform":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sku-versions","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"price":{"type":"string"},"source":{"type":"string"},"codeSku":{"type":"string"},"picture":{"type":"string"},"product":{"type":"string"},"version":{"type":"number"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"properties":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sub-categories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"order":{"type":"number"},"active":{"type":"boolean"},"parent":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"totalProduct":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"products","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"desc":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"temp":{"type":"string"},"brand":{"type":"string"},"order":{"type":"string"},"price":{"type":"string"},"score":{"type":"string"},"active":{"type":"boolean"},"author":{"type":"string"},"guides":{"type":"array"},"photos":{"type":"array"},"videos":{"type":"array"},"quantity":{"type":"number"},"createdAt":{"type":"string"},"documents":{"type":"array"},"shareDesc":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"inactiveAt":{"type":"string"},"properties":{"type":"array"},"reActiveAt":{"type":"string"},"inventories":{"type":"array"},"displaySKUId":{"type":"string"},"isOutOfStock":{"type":"boolean"},"pricePercent":{"type":"string"},"searchString":{"type":"string"},"subCategories":{"type":"array"},"inactiveReason":{"type":"string"},"propertiesMain":{"type":"string"},"shareStatistic":{"type":"string"},"updateActiveId":{"type":"string"},"canIssueInvoice":{"type":"boolean"},"pendingInactive":{"type":"boolean"},"reviewStatistic":{"type":"string"},"highlightProperties":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"promotion-orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"promotion":{"type":"string"},"updatedAt":{"type":"string"},"cashbackAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"orderPriceTotal":{"type":"number"},"quantityPromotion":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"promotions","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"rest":{"type":"number"},"type":{"type":"string"},"endAt":{"type":"string"},"title":{"type":"string"},"active":{"type":"boolean"},"options":{"type":"string"},"startAt":{"type":"string"},"applyFor":{"type":"string"},"quantity":{"type":"number"},"articleID":{"type":"string"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"conditionForUser":{"type":"string"},"supplierCommission":{"type":"string"},"isUnlimitedQuantity":{"type":"boolean"},"isApplyForOrderWholesale":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"referrals","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"invitee":{"type":"string"},"createdAt":{"type":"string"},"fromSystem":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"social-post-views","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"clientIP":{"type":"string"},"lastViewAt":{"type":"string"},"socialPost":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"social-posts","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"tags":{"type":"array"},"isPin":{"type":"boolean"},"order":{"type":"number"},"title":{"type":"string"},"author":{"type":"string"},"cities":{"type":"array"},"photos":{"type":"array"},"reason":{"type":"string"},"status":{"type":"string"},"videos":{"type":"array"},"content":{"type":"string"},"isTimer":{"type":"boolean"},"startAt":{"type":"string"},"products":{"type":"array"},"createdAt":{"type":"string"},"hasUpdate":{"type":"boolean"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"fromSystem":{"type":"boolean"},"contributor":{"type":"string"},"publishedAt":{"type":"string"},"searchString":{"type":"string"},"updatedCount":{"type":"number"},"shareStatistic":{"type":"string"},"enableNotificationForContributor":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"suppliers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"tax":{"type":"string"},"bank":{"type":"string"},"logo":{"type":"string"},"name":{"type":"string"},"active":{"type":"boolean"},"balance":{"type":"string"},"checkSum":{"type":"string"},"checksum":{"type":"string"},"location":{"type":"string"},"contracts":{"type":"array"},"createdAt":{"type":"string"},"documents":{"type":"array"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"inventories":{"type":"array"},"identifyCode":{"type":"number"},"searchString":{"type":"string"},"reconcileDays":{"type":"array"},"contractNumber":{"type":"string"},"reviewStatistic":{"type":"string"},"contractSignedAt":{"type":"string"},"emailForInvoices":{"type":"array"},"contractExpiredAt":{"type":"string"},"sendInvoiceRequestEveryDay":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-activities","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"team":{"type":"string"},"action":{"type":"string"},"options":{"type":"string"},"targetId":{"type":"string"},"createdAt":{"type":"string"},"createdBy":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-bonus","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"team":{"type":"string"},"user":{"type":"string"},"status":{"type":"string"},"percent":{"type":"number"},"targetId":{"type":"string"},"teamName":{"type":"string"},"createdAt":{"type":"string"},"teamLevel":{"type":"number"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"teamMember":{"type":"string"},"teamPromotion":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-members","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"role":{"type":"string"},"team":{"type":"string"},"user":{"type":"string"},"isLeft":{"type":"boolean"},"leftAt":{"type":"string"},"joinedAt":{"type":"string"},"roleCode":{"type":"number"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"isRemovedBy":{"type":"string"},"searchString":{"type":"string"},"statisticMonth":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"teams","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"desc":{"type":"string"},"logo":{"type":"string"},"name":{"type":"string"},"temp":{"type":"string"},"level":{"type":"string"},"active":{"type":"boolean"},"isFull":{"type":"boolean"},"cityCode":{"type":"number"},"adminUser":{"type":"string"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"inviteLink":{"type":"string"},"information":{"type":"string"},"searchString":{"type":"string"},"inactiveReason":{"type":"string"},"changingNameCount":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"tracking-product-shares","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"source":{"type":"string"},"options":{"type":"string"},"products":{"type":"string"},"createdAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"user-identifications","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"dob":{"type":"string"},"name":{"type":"string"},"note":{"type":"string"},"type":{"type":"string"},"user":{"type":"string"},"gender":{"type":"string"},"nation":{"type":"string"},"number":{"type":"string"},"status":{"type":"string"},"address":{"type":"string"},"backSide":{"type":"string"},"birthday":{"type":"string"},"createdAt":{"type":"string"},"frontSide":{"type":"string"},"issueDate":{"type":"string"},"updatedAt":{"type":"string"},"expiryDate":{"type":"string"},"issuePlace":{"type":"string"},"timeUpdate":{"type":"number"},"detectInformation":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"users","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"team":{"type":"string"},"zalo":{"type":"string"},"email":{"type":"string"},"phone":{"type":"string"},"avatar":{"type":"string"},"banned":{"type":"boolean"},"segment":{"type":"string"},"facebook":{"type":"string"},"hasOrder":{"type":"boolean"},"referral":{"type":"string"},"segments":{"type":"array"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"membership":{"type":"string"},"registerFrom":{"type":"string"},"searchString":{"type":"string"},"isUpdatedInfo":{"type":"boolean"},"identification":{"type":"string"},"lastActivatedAt":{"type":"string"},"lastViewNotificationAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"wards","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"city":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"tncId":{"type":"number"},"cityId":{"type":"number"},"source":{"type":"string"},"tncCode":{"type":"string"},"district":{"type":"string"},"gidoCode":{"type":"number"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"districtId":{"type":"number"},"provinceId":{"type":"number"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]}]} \ No newline at end of file +{"streams":[{"stream":{"name":"admin-tags","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"active":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"cash-flows","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"hash":{"type":"string"},"user":{"type":"string"},"value":{"type":"number"},"action":{"type":"string"},"options":{"type":"string"},"category":{"type":"string"},"targetId":{"type":"string"},"createdAt":{"type":"string"},"isAudited":{"type":"boolean"},"updatedAt":{"type":"string"},"newBalance":{"type":"number"},"oldBalance":{"type":"number"},"targetType":{"type":"string"},"isProcessed":{"type":"boolean"},"processStatus":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"cities","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"order":{"type":"number"},"tncId":{"type":"number"},"region":{"type":"string"},"source":{"type":"string"},"tempId":{"type":"number"},"tncCode":{"type":"string"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"customers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"user":{"type":"string"},"email":{"type":"string"},"phone":{"type":"string"},"source":{"type":"string"},"location":{"type":"array"},"createdAt":{"type":"string"},"isDeleted":{"type":"boolean"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"districts","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"city":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"tncId":{"type":"number"},"cityId":{"type":"number"},"source":{"type":"string"},"tncCode":{"type":"string"},"gidoCode":{"type":"number"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"event-rewards","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"cash":{"type":"number"},"type":{"type":"string"},"user":{"type":"string"},"event":{"type":"string"},"status":{"type":"string"},"options":{"type":"string"},"createdAt":{"type":"string"},"updatedat":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"events","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"desc":{"type":"string"},"icon":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"cover":{"type":"string"},"endAt":{"type":"string"},"active":{"type":"boolean"},"notice":{"type":"string"},"screen":{"type":"string"},"article":{"type":"string"},"options":{"type":"string"},"startAt":{"type":"string"},"segments":{"type":"string"},"createdAt":{"type":"string"},"groupType":{"type":"string"},"suppliers":{"type":"array"},"updatedAt":{"type":"string"},"information":{"type":"array"},"displayEndAt":{"type":"string"},"isApplyForAll":{"type":"boolean"},"displayStartAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"inventories","json_schema":{"type":"object","properties":{"id":{"type":"string"},"_id":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"active":{"type":"boolean"},"contact":{"type":"string"},"partner":{"type":"string"},"checksum":{"type":"string"},"location":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"minimumValue":{"type":"number"},"searchString":{"type":"string"},"paymentMethods":{"type":"string"},"canIssueInvoice":{"type":"boolean"},"deliveryMethods":{"type":"array"},"canAutoSendEmail":{"type":"boolean"},"invoiceDeliveryMethod":{"type":"string"},"doesSupportSellyExpress":{"type":"boolean"},"priorityDeliveryServiceCodes":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"color":{"type":"string"},"guide":{"type":"string"},"level":{"type":"number"},"sales":{"type":"number"},"active":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"updatedat":{"type":"string"},"transaction":{"type":"number"},"bonusPercent":{"type":"number"},"bonuspercent":{"type":"number"},"transactionMinValue":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership-histories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"new":{"type":"string"},"old":{"type":"string"},"type":{"type":"string"},"user":{"type":"string"},"createdAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership-orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"status":{"type":"string"},"percent":{"type":"number"},"targetID":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"membershipName":{"type":"string"},"membershipLevel":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-deliveries","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"service":{"type":"string"},"customer":{"type":"string"},"delivery":{"type":"string"},"location":{"type":"string"},"createdAt":{"type":"string"},"promotion":{"type":"string"},"updatedAt":{"type":"string"},"trackingCode":{"type":"string"},"trackingTime":{"type":"string"},"isAdminChange":{"type":"boolean"},"trackingOrderCode":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-histories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"order":{"type":"string"},"status":{"type":"string"},"createdAt":{"type":"string"},"timestamp":{"type":"number"},"updatedAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-items","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"date":{"type":"string"},"user":{"type":"string"},"brand":{"type":"string"},"order":{"type":"string"},"price":{"type":"string"},"status":{"type":"string"},"teamId":{"type":"string"},"product":{"type":"string"},"voucher":{"type":"string"},"customer":{"type":"string"},"quantity":{"type":"number"},"createdAt":{"type":"string"},"inventory":{"type":"string"},"totalSell":{"type":"number"},"updatedAt":{"type":"string"},"cashbackAt":{"type":"string"},"promotions":{"type":"array"},"totalPrice":{"type":"number"},"deliveredAt":{"type":"string"},"teamMemberId":{"type":"string"},"isAssignCoupon":{"type":"boolean"},"totalPromotion":{"type":"number"},"inWholesaleRange":{"type":"boolean"},"voucherCashTotal":{"type":"number"},"wholesaleRangeId":{"type":"string"},"totalWholesaleBonus":{"type":"number"},"totalSellyWholesaleBonus":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"date":{"type":"string"},"hour":{"type":"number"},"note":{"type":"string"},"tags":{"type":"array"},"user":{"type":"string"},"price":{"type":"string"},"banned":{"type":"boolean"},"reason":{"type":"string"},"source":{"type":"string"},"status":{"type":"string"},"teamId":{"type":"string"},"payment":{"type":"string"},"remarks":{"type":"string"},"customer":{"type":"string"},"delivery":{"type":"string"},"isCalled":{"type":"boolean"},"merchant":{"type":"string"},"pickupAt":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"inventory":{"type":"string"},"invoiceId":{"type":"string"},"isDeleted":{"type":"boolean"},"promotion":{"type":"string"},"requestId":{"type":"string"},"restockAt":{"type":"string"},"sendEmail":{"type":"string"},"totalItem":{"type":"number"},"updatedAt":{"type":"string"},"userAgent":{"type":"string"},"wholesale":{"type":"boolean"},"approvedAt":{"type":"string"},"cashbackAt":{"type":"string"},"isPreorder":{"type":"boolean"},"isReviewed":{"type":"boolean"},"rejectedAt":{"type":"string"},"deliveredAt":{"type":"string"},"deliveringAt":{"type":"string"},"deliveryCode":{"type":"string"},"searchString":{"type":"string"},"staffApprove":{"type":"string"},"teamMemberId":{"type":"string"},"processStatus":{"type":"string"},"hookTimeLastAt":{"type":"string"},"isAssignCoupon":{"type":"boolean"},"isAutoApproved":{"type":"boolean"},"outboundRequest":{"type":"string"},"trackingCodeURL":{"type":"string"},"waitingCancelBy":{"type":"string"},"isChangeDelivery":{"type":"boolean"},"estimateCashbackAt":{"type":"string"},"fromNewActiveBuyer":{"type":"boolean"},"isWaitingCancelled":{"type":"boolean"},"fromNewActiveSeller":{"type":"boolean"},"waitingCancelReason":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-categories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"icon":{"type":"string"},"name":{"type":"string"},"color":{"type":"string"},"order":{"type":"number"},"active":{"type":"boolean"},"covers":{"type":"array"},"featured":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"totalProduct":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sku","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"brand":{"type":"string"},"cover":{"type":"string"},"price":{"type":"string"},"active":{"type":"boolean"},"author":{"type":"string"},"source":{"type":"string"},"display":{"type":"boolean"},"picture":{"type":"string"},"product":{"type":"string"},"version":{"type":"number"},"groupSku":{"type":"string"},"unitCode":{"type":"string"},"createdAt":{"type":"string"},"restockAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"couponInfo":{"type":"string"},"properties":{"type":"array"},"canPreorder":{"type":"boolean"},"supplierSku":{"type":"string"},"isOutOfStock":{"type":"boolean"},"pricePercent":{"type":"string"},"searchString":{"type":"string"},"displayInventory":{"type":"boolean"},"showRemainingQuantity":{"type":"boolean"},"quantity_aibyte_transform":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sku-versions","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"price":{"type":"string"},"source":{"type":"string"},"codeSku":{"type":"string"},"picture":{"type":"string"},"product":{"type":"string"},"version":{"type":"number"},"createdAt":{"type":"string"},"properties":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sub-categories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"order":{"type":"number"},"active":{"type":"boolean"},"parent":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"totalProduct":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"products","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"desc":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"temp":{"type":"string"},"brand":{"type":"string"},"order":{"type":"string"},"price":{"type":"string"},"score":{"type":"string"},"active":{"type":"boolean"},"author":{"type":"string"},"guides":{"type":"array"},"photos":{"type":"array"},"videos":{"type":"array"},"quantity":{"type":"number"},"createdAt":{"type":"string"},"documents":{"type":"array"},"shareDesc":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"properties":{"type":"array"},"reActiveAt":{"type":"string"},"inventories":{"type":"array"},"isOutOfStock":{"type":"boolean"},"pricePercent":{"type":"string"},"searchString":{"type":"string"},"subCategories":{"type":"array"},"inactiveReason":{"type":"string"},"propertiesMain":{"type":"string"},"shareStatistic":{"type":"string"},"canIssueInvoice":{"type":"boolean"},"pendingInactive":{"type":"boolean"},"reviewStatistic":{"type":"string"},"pendingInactiveAt":{"type":"string"},"highlightProperties":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"promotion-orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"promotion":{"type":"string"},"updatedAt":{"type":"string"},"cashbackAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"quantityPromotion":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"promotions","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"rest":{"type":"number"},"type":{"type":"string"},"endAt":{"type":"string"},"title":{"type":"string"},"active":{"type":"boolean"},"options":{"type":"string"},"startAt":{"type":"string"},"applyFor":{"type":"string"},"quantity":{"type":"number"},"articleID":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"sharePercent":{"type":"string"},"conditionForUser":{"type":"string"},"isUnlimitedQuantity":{"type":"boolean"},"isApplyForOrderWholesale":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"referrals","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"invitee":{"type":"string"},"createdAt":{"type":"string"},"fromSystem":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"social-post-views","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"clientIP":{"type":"string"},"lastViewAt":{"type":"string"},"socialPost":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"social-posts","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"tags":{"type":"array"},"isPin":{"type":"boolean"},"order":{"type":"number"},"title":{"type":"string"},"author":{"type":"string"},"cities":{"type":"array"},"photos":{"type":"array"},"reason":{"type":"string"},"status":{"type":"string"},"videos":{"type":"array"},"content":{"type":"string"},"isTimer":{"type":"boolean"},"startAt":{"type":"string"},"products":{"type":"array"},"createdAt":{"type":"string"},"hasUpdate":{"type":"boolean"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"fromSystem":{"type":"boolean"},"contributor":{"type":"string"},"publishedAt":{"type":"string"},"searchString":{"type":"string"},"updatedCount":{"type":"number"},"shareStatistic":{"type":"string"},"enableNotificationForContributor":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"suppliers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"tax":{"type":"string"},"bank":{"type":"string"},"logo":{"type":"string"},"name":{"type":"string"},"active":{"type":"boolean"},"checkSum":{"type":"string"},"checksum":{"type":"string"},"location":{"type":"string"},"contracts":{"type":"array"},"createdAt":{"type":"string"},"documents":{"type":"array"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"inventories":{"type":"array"},"identifyCode":{"type":"number"},"searchString":{"type":"string"},"contractNumber":{"type":"string"},"reviewStatistic":{"type":"string"},"contractSignedAt":{"type":"string"},"emailForInvoices":{"type":"array"},"contractExpiredAt":{"type":"string"},"sendInvoiceRequestEveryDay":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-activities","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"team":{"type":"string"},"action":{"type":"string"},"options":{"type":"string"},"targetId":{"type":"string"},"createdAt":{"type":"string"},"createdBy":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-bonus","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"team":{"type":"string"},"user":{"type":"string"},"status":{"type":"string"},"percent":{"type":"number"},"targetId":{"type":"string"},"teamName":{"type":"string"},"createdAt":{"type":"string"},"teamLevel":{"type":"number"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"teamMember":{"type":"string"},"teamPromotion":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-members","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"role":{"type":"string"},"team":{"type":"string"},"user":{"type":"string"},"isLeft":{"type":"boolean"},"leftAt":{"type":"string"},"joinedAt":{"type":"string"},"roleCode":{"type":"number"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"isRemovedBy":{"type":"string"},"firstOrderAt":{"type":"string"},"searchString":{"type":"string"},"isMemberHomegrown":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"teams","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"desc":{"type":"string"},"logo":{"type":"string"},"name":{"type":"string"},"level":{"type":"string"},"active":{"type":"boolean"},"isFull":{"type":"boolean"},"cityCode":{"type":"number"},"adminUser":{"type":"string"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"inviteLink":{"type":"string"},"information":{"type":"string"},"searchString":{"type":"string"},"inactiveReason":{"type":"string"},"changingNameCount":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"user-identifications","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"dob":{"type":"string"},"name":{"type":"string"},"note":{"type":"string"},"type":{"type":"string"},"user":{"type":"string"},"gender":{"type":"string"},"nation":{"type":"string"},"number":{"type":"string"},"status":{"type":"string"},"address":{"type":"string"},"dobDate":{"type":"string"},"backSide":{"type":"string"},"birthday":{"type":"string"},"createdAt":{"type":"string"},"frontSide":{"type":"string"},"issueDate":{"type":"string"},"updatedAt":{"type":"string"},"expiryDate":{"type":"string"},"issuePlace":{"type":"string"},"timeUpdate":{"type":"number"},"detectInformation":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"users","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"team":{"type":"string"},"zalo":{"type":"string"},"email":{"type":"string"},"phone":{"type":"string"},"avatar":{"type":"string"},"banned":{"type":"boolean"},"segment":{"type":"string"},"facebook":{"type":"string"},"hasOrder":{"type":"boolean"},"referral":{"type":"string"},"segments":{"type":"array"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"membership":{"type":"string"},"lastOrderAt":{"type":"string"},"firstOrderAt":{"type":"string"},"registerFrom":{"type":"string"},"searchString":{"type":"string"},"adjustTracker":{"type":"string"},"isUpdatedInfo":{"type":"boolean"},"identification":{"type":"string"},"lastActivatedAt":{"type":"string"},"lastViewNotificationAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"voucher-bonus","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"userVoucher":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"vouchers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"desc":{"type":"string"},"logo":{"type":"string"},"type":{"type":"string"},"endAt":{"type":"string"},"title":{"type":"string"},"active":{"type":"boolean"},"covers":{"type":"array"},"options":{"type":"string"},"startAt":{"type":"string"},"applyFor":{"type":"string"},"articleID":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"isApplyForOrderWholesale":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"wards","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"city":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"tncId":{"type":"number"},"cityId":{"type":"number"},"source":{"type":"string"},"tncCode":{"type":"string"},"district":{"type":"string"},"gidoCode":{"type":"number"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"districtId":{"type":"number"},"provinceId":{"type":"number"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]}]} \ No newline at end of file diff --git a/macros/configuration.sql b/macros/configuration.sql new file mode 100644 index 0000000..a599ab3 --- /dev/null +++ b/macros/configuration.sql @@ -0,0 +1,14 @@ +{%- macro redshift_super_type() -%} + {%- if not execute -%} + {{ return("") }} + {%- endif -%} + + {%- set table_schema, _, table_name = var("models_to_source")[this.identifier].partition(".") -%} + + {%- call statement("get_column_type", fetch_result=True) -%} + select data_type from SVV_COLUMNS where table_name = '{{ table_name }}' and column_name = '{{ var("json_column") }}' and table_schema = '{{ table_schema }}'; + {%- endcall -%} + + {%- set column_type = load_result("get_column_type")["data"][0][0] -%} + {{ return(column_type == "super") }} +{%- endmacro -%} diff --git a/macros/cross_db_utils/array.sql b/macros/cross_db_utils/array.sql index 9072da2..35df407 100644 --- a/macros/cross_db_utils/array.sql +++ b/macros/cross_db_utils/array.sql @@ -104,8 +104,19 @@ {% macro default__unnest_cte(from_table, stream_name, column_col) -%}{%- endmacro %} -{# -- based on https://blog.getdbt.com/how-to-unnest-arrays-in-redshift/ #} {% macro redshift__unnest_cte(from_table, stream_name, column_col) -%} + + {# -- based on https://docs.aws.amazon.com/redshift/latest/dg/query-super.html #} + {% if redshift_super_type() -%} + with joined as ( + select + table_alias._airbyte_{{ stream_name }}_hashid as _airbyte_hashid, + _airbyte_nested_data + from {{ from_table }} as table_alias, table_alias.{{ column_col }} as _airbyte_nested_data + ) + {%- else -%} + + {# -- based on https://blog.getdbt.com/how-to-unnest-arrays-in-redshift/ #} {%- if not execute -%} {{ return('') }} {% endif %} @@ -134,6 +145,7 @@ joined as ( -- to the number of items in {{ from_table }}.{{ column_col }} where numbers.generated_number <= json_array_length({{ column_col }}, true) ) + {%- endif %} {%- endmacro %} {% macro mysql__unnest_cte(from_table, stream_name, column_col) -%} diff --git a/macros/cross_db_utils/columns.sql b/macros/cross_db_utils/columns.sql new file mode 100644 index 0000000..0b695c1 --- /dev/null +++ b/macros/cross_db_utils/columns.sql @@ -0,0 +1,16 @@ +{% macro redshift__alter_column_type(relation, column_name, new_column_type) -%} + + {%- set tmp_column = column_name + "__dbt_alter" -%} + + {% call statement('alter_column_type') %} + alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; + {% if new_column_type.lower() == "super" %} + update {{ relation }} set {{ adapter.quote(tmp_column) }} = JSON_PARSE({{ adapter.quote(column_name) }}); + {% else %} + update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; + {% endif %} + alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade; + alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} + {% endcall %} + +{% endmacro %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index 080aea5..07600de 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -9,7 +9,11 @@ {% endmacro %} {%- macro redshift__type_json() -%} + {%- if redshift_super_type() -%} + super + {%- else -%} varchar + {%- endif -%} {%- endmacro -%} {% macro postgres__type_json() %} diff --git a/macros/cross_db_utils/json_operations.sql b/macros/cross_db_utils/json_operations.sql index cf52f24..e1e5443 100644 --- a/macros/cross_db_utils/json_operations.sql +++ b/macros/cross_db_utils/json_operations.sql @@ -43,11 +43,12 @@ {%- endmacro %} {% macro redshift__format_json_path(json_path_list) -%} + {%- set quote = '"' if redshift_super_type() else "'" -%} {%- set str_list = [] -%} {%- for json_path in json_path_list -%} - {%- if str_list.append(json_path.replace("'", "''")) -%} {%- endif -%} + {%- if str_list.append(json_path.replace(quote, quote + quote)) -%} {%- endif -%} {%- endfor -%} - {{ "'" ~ str_list|join("','") ~ "'" }} + {{ quote ~ str_list|join(quote + "," + quote) ~ quote }} {%- endmacro %} {% macro snowflake__format_json_path(json_path_list) -%} @@ -114,11 +115,14 @@ {%- endmacro %} {% macro redshift__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} - {%- if from_table|string() == '' %} + {%- if from_table|string() != '' -%} + {%- set json_column = from_table|string() + "." + json_column|string() -%} + {%- endif -%} + {%- if redshift_super_type() -%} + case when {{ json_column }}.{{ format_json_path(json_path_list) }} != '' then {{ json_column }}.{{ format_json_path(json_path_list) }} end + {%- else -%} case when json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) end - {% else %} - case when json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) end - {% endif -%} + {%- endif -%} {%- endmacro %} {% macro snowflake__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} @@ -168,7 +172,11 @@ {%- endmacro %} {% macro redshift__json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} + {%- if redshift_super_type() -%} + case when {{ json_column }}.{{ format_json_path(json_path_list) }} != '' then {{ json_column }}.{{ format_json_path(json_path_list) }} end + {%- else -%} case when json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) end + {%- endif -%} {%- endmacro %} {% macro snowflake__json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} @@ -210,7 +218,11 @@ {%- endmacro %} {% macro redshift__json_extract_array(json_column, json_path_list, normalized_json_path) -%} + {%- if redshift_super_type() -%} + {{ json_column }}.{{ format_json_path(json_path_list) }} + {%- else -%} json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) + {%- endif -%} {%- endmacro %} {% macro snowflake__json_extract_array(json_column, json_path_list, normalized_json_path) -%} diff --git a/macros/cross_db_utils/type_conversions.sql b/macros/cross_db_utils/type_conversions.sql index 1ad849b..ad3d756 100644 --- a/macros/cross_db_utils/type_conversions.sql +++ b/macros/cross_db_utils/type_conversions.sql @@ -33,6 +33,31 @@ cast({{ array_column }} as {{dbt_utils.type_string()}}) {%- endmacro %} +{% macro redshift__array_to_string(array_column) -%} + {% if redshift_super_type() -%} + json_serialize({{array_column}}) + {%- else -%} + {{ array_column }} + {%- endif %} +{%- endmacro %} + +{# object_to_string ------------------------------------------------- #} +{% macro object_to_string(object_column) -%} + {{ adapter.dispatch('object_to_string')(object_column) }} +{%- endmacro %} + +{% macro default__object_to_string(object_column) -%} + {{ object_column }} +{%- endmacro %} + +{% macro redshift__object_to_string(object_column) -%} + {% if redshift_super_type() -%} + json_serialize({{object_column}}) + {%- else -%} + {{ object_column }} + {%- endif %} +{%- endmacro %} + {# cast_to_boolean ------------------------------------------------- #} {% macro cast_to_boolean(field) -%} {{ adapter.dispatch('cast_to_boolean')(field) }} @@ -49,7 +74,11 @@ {# -- Redshift does not support converting string directly to boolean, it must go through int first #} {% macro redshift__cast_to_boolean(field) -%} + {% if redshift_super_type() -%} + cast({{ field }} as boolean) + {%- else -%} cast(decode({{ field }}, 'true', '1', 'false', '0')::integer as boolean) + {%- endif %} {%- endmacro %} {# -- MS SQL Server does not support converting string directly to boolean, it must be casted as bit #} @@ -70,3 +99,7 @@ {%- macro default__empty_string_to_null(field) -%} nullif({{ field }}, '') {%- endmacro %} + +{%- macro redshift__empty_string_to_null(field) -%} + nullif({{ field }}::varchar, '') +{%- endmacro %} diff --git a/models/generated/airbyte_ctes/unibag/voucher_bonus_ab1.sql b/models/generated/airbyte_ctes/unibag/voucher_bonus_ab1.sql new file mode 100644 index 0000000..2f52391 --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/voucher_bonus_ab1.sql @@ -0,0 +1,26 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('unibag', '_airbyte_raw_voucher_bonus') }} +select + {{ json_extract_scalar('_airbyte_data', ['_id'], ['_id']) }} as _id, + {{ json_extract_scalar('_airbyte_data', ['user'], ['user']) }} as {{ adapter.quote('user') }}, + {{ json_extract_scalar('_airbyte_data', ['order'], ['order']) }} as {{ adapter.quote('order') }}, + {{ json_extract_scalar('_airbyte_data', ['title'], ['title']) }} as title, + {{ json_extract_scalar('_airbyte_data', ['status'], ['status']) }} as status, + {{ json_extract_scalar('_airbyte_data', ['createdAt'], ['createdAt']) }} as createdat, + {{ json_extract_scalar('_airbyte_data', ['updatedAt'], ['updatedAt']) }} as updatedat, + {{ json_extract_scalar('_airbyte_data', ['commission'], ['commission']) }} as commission, + {{ json_extract_scalar('_airbyte_data', ['isRejected'], ['isRejected']) }} as isrejected, + {{ json_extract_scalar('_airbyte_data', ['userVoucher'], ['userVoucher']) }} as uservoucher, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('unibag', '_airbyte_raw_voucher_bonus') }} as table_alias +-- voucher_bonus +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/voucher_bonus_ab2.sql b/models/generated/airbyte_ctes/unibag/voucher_bonus_ab2.sql new file mode 100644 index 0000000..cefe14d --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/voucher_bonus_ab2.sql @@ -0,0 +1,26 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('voucher_bonus_ab1') }} +select + cast(_id as {{ dbt_utils.type_string() }}) as _id, + cast({{ adapter.quote('user') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('user') }}, + cast({{ adapter.quote('order') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('order') }}, + cast(title as {{ dbt_utils.type_string() }}) as title, + cast(status as {{ dbt_utils.type_string() }}) as status, + cast(createdat as {{ dbt_utils.type_string() }}) as createdat, + cast(updatedat as {{ dbt_utils.type_string() }}) as updatedat, + cast(commission as {{ dbt_utils.type_float() }}) as commission, + {{ cast_to_boolean('isrejected') }} as isrejected, + cast(uservoucher as {{ dbt_utils.type_string() }}) as uservoucher, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('voucher_bonus_ab1') }} +-- voucher_bonus +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/voucher_bonus_ab3.sql b/models/generated/airbyte_ctes/unibag/voucher_bonus_ab3.sql new file mode 100644 index 0000000..ef84487 --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/voucher_bonus_ab3.sql @@ -0,0 +1,26 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +-- depends_on: {{ ref('voucher_bonus_ab2') }} +select + {{ dbt_utils.surrogate_key([ + '_id', + adapter.quote('user'), + adapter.quote('order'), + 'title', + 'status', + 'createdat', + 'updatedat', + 'commission', + boolean_to_string('isrejected'), + 'uservoucher', + ]) }} as _airbyte_voucher_bonus_hashid, + tmp.* +from {{ ref('voucher_bonus_ab2') }} tmp +-- voucher_bonus +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/vouchers_ab1.sql b/models/generated/airbyte_ctes/unibag/vouchers_ab1.sql new file mode 100644 index 0000000..54b21a5 --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/vouchers_ab1.sql @@ -0,0 +1,33 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('unibag', '_airbyte_raw_vouchers') }} +select + {{ json_extract_scalar('_airbyte_data', ['_id'], ['_id']) }} as _id, + {{ json_extract_scalar('_airbyte_data', ['code'], ['code']) }} as code, + {{ json_extract_scalar('_airbyte_data', ['desc'], ['desc']) }} as {{ adapter.quote('desc') }}, + {{ json_extract_scalar('_airbyte_data', ['logo'], ['logo']) }} as logo, + {{ json_extract_scalar('_airbyte_data', ['type'], ['type']) }} as {{ adapter.quote('type') }}, + {{ json_extract_scalar('_airbyte_data', ['endAt'], ['endAt']) }} as endat, + {{ json_extract_scalar('_airbyte_data', ['title'], ['title']) }} as title, + {{ json_extract_scalar('_airbyte_data', ['active'], ['active']) }} as active, + {{ json_extract_array('_airbyte_data', ['covers'], ['covers']) }} as covers, + {{ json_extract_scalar('_airbyte_data', ['options'], ['options']) }} as {{ adapter.quote('options') }}, + {{ json_extract_scalar('_airbyte_data', ['startAt'], ['startAt']) }} as startat, + {{ json_extract_scalar('_airbyte_data', ['applyFor'], ['applyFor']) }} as applyfor, + {{ json_extract_scalar('_airbyte_data', ['articleID'], ['articleID']) }} as articleid, + {{ json_extract_scalar('_airbyte_data', ['createdAt'], ['createdAt']) }} as createdat, + {{ json_extract_scalar('_airbyte_data', ['updatedAt'], ['updatedAt']) }} as updatedat, + {{ json_extract_scalar('_airbyte_data', ['searchString'], ['searchString']) }} as searchstring, + {{ json_extract_scalar('_airbyte_data', ['isApplyForOrderWholesale'], ['isApplyForOrderWholesale']) }} as isapplyfororderwholesale, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('unibag', '_airbyte_raw_vouchers') }} as table_alias +-- vouchers +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/vouchers_ab2.sql b/models/generated/airbyte_ctes/unibag/vouchers_ab2.sql new file mode 100644 index 0000000..f48ce06 --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/vouchers_ab2.sql @@ -0,0 +1,33 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('vouchers_ab1') }} +select + cast(_id as {{ dbt_utils.type_string() }}) as _id, + cast(code as {{ dbt_utils.type_string() }}) as code, + cast({{ adapter.quote('desc') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('desc') }}, + cast(logo as {{ dbt_utils.type_string() }}) as logo, + cast({{ adapter.quote('type') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('type') }}, + cast(endat as {{ dbt_utils.type_string() }}) as endat, + cast(title as {{ dbt_utils.type_string() }}) as title, + {{ cast_to_boolean('active') }} as active, + covers, + cast({{ adapter.quote('options') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('options') }}, + cast(startat as {{ dbt_utils.type_string() }}) as startat, + cast(applyfor as {{ dbt_utils.type_string() }}) as applyfor, + cast(articleid as {{ dbt_utils.type_string() }}) as articleid, + cast(createdat as {{ dbt_utils.type_string() }}) as createdat, + cast(updatedat as {{ dbt_utils.type_string() }}) as updatedat, + cast(searchstring as {{ dbt_utils.type_string() }}) as searchstring, + {{ cast_to_boolean('isapplyfororderwholesale') }} as isapplyfororderwholesale, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('vouchers_ab1') }} +-- vouchers +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/vouchers_ab3.sql b/models/generated/airbyte_ctes/unibag/vouchers_ab3.sql new file mode 100644 index 0000000..2fa0827 --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/vouchers_ab3.sql @@ -0,0 +1,33 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +-- depends_on: {{ ref('vouchers_ab2') }} +select + {{ dbt_utils.surrogate_key([ + '_id', + 'code', + adapter.quote('desc'), + 'logo', + adapter.quote('type'), + 'endat', + 'title', + boolean_to_string('active'), + array_to_string('covers'), + adapter.quote('options'), + 'startat', + 'applyfor', + 'articleid', + 'createdat', + 'updatedat', + 'searchstring', + boolean_to_string('isapplyfororderwholesale'), + ]) }} as _airbyte_vouchers_hashid, + tmp.* +from {{ ref('vouchers_ab2') }} tmp +-- vouchers +where 1 = 1 + diff --git a/models/generated/airbyte_tables/unibag/voucher_bonus.sql b/models/generated/airbyte_tables/unibag/voucher_bonus.sql new file mode 100644 index 0000000..3c96084 --- /dev/null +++ b/models/generated/airbyte_tables/unibag/voucher_bonus.sql @@ -0,0 +1,26 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "unibag", + tags = [ "top-level" ] +) }} +-- Final base SQL model +-- depends_on: {{ ref('voucher_bonus_ab3') }} +select + _id, + {{ adapter.quote('user') }} AS seller_id, + {{ adapter.quote('order') }} AS order_id, + title, + status, + commission, + isrejected AS is_rejected, + createdat::timestamp AS created_at, + updatedat::timestamp AS updated_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_voucher_bonus_hashid +from {{ ref('voucher_bonus_ab3') }} +-- voucher_bonus from {{ source('unibag', '_airbyte_raw_voucher_bonus') }} +where 1 = 1 + diff --git a/models/generated/airbyte_tables/unibag/vouchers.sql b/models/generated/airbyte_tables/unibag/vouchers.sql new file mode 100644 index 0000000..32528b7 --- /dev/null +++ b/models/generated/airbyte_tables/unibag/vouchers.sql @@ -0,0 +1,30 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "unibag", + tags = [ "top-level" ] +) }} +-- Final base SQL model +-- depends_on: {{ ref('vouchers_ab3') }} +select + _id, + code, + {{ adapter.quote('type') }}, + title, + active, + cast({{ adapter.quote('options') }}::json->>'milestones' AS jsonb) AS options_milestones, + cast({{ adapter.quote('options') }}::json->>'transactionMinValue' AS numeric) AS options_transaction_min_value, + articleid AS article_id, + isapplyfororderwholesale AS is_apply_for_order_wholesale, + startat::timestamp AS start_at, + endat::timestamp AS end_at, + createdat::timestamp AS created_at, + updatedat::timestamp AS updated_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_vouchers_hashid +from {{ ref('vouchers_ab3') }} +-- vouchers from {{ source('unibag', '_airbyte_raw_vouchers') }} +where 1 = 1 + diff --git a/models/generated/sources.yml b/models/generated/sources.yml index f424af8..8fdcd74 100644 --- a/models/generated/sources.yml +++ b/models/generated/sources.yml @@ -36,7 +36,8 @@ sources: - name: _airbyte_raw_team_bonus - name: _airbyte_raw_team_members - name: _airbyte_raw_teams - - name: _airbyte_raw_tracking_product_shares - name: _airbyte_raw_user_identifications - name: _airbyte_raw_users + - name: _airbyte_raw_voucher_bonus + - name: _airbyte_raw_vouchers - name: _airbyte_raw_wards