From 250f61873c5f165f19a34c4a191548912944efc3 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 13 Apr 2022 14:06:23 +0700 Subject: [PATCH] update --- Makefile | 2 +- dbt_project.yml | 32 +++++++------- destination_catalog.json | 2 +- destination_config.json | 2 +- macros/cross_db_utils/array.sql | 5 +++ macros/cross_db_utils/json_operations.sql | 8 ++-- macros/cross_db_utils/type_conversions.sql | 5 +++ .../unibag/social_post_views_ab1.sql | 21 +++++++++ .../unibag/social_post_views_ab2.sql | 21 +++++++++ .../unibag/social_post_views_ab3.sql | 21 +++++++++ .../airbyte_ctes/unibag/social_posts_ab1.sql | 43 +++++++++++++++++++ .../airbyte_ctes/unibag/social_posts_ab2.sql | 43 +++++++++++++++++++ .../airbyte_ctes/unibag/social_posts_ab3.sql | 43 +++++++++++++++++++ .../unibag/tracking_product_shares_ab1.sql | 22 ++++++++++ .../unibag/tracking_product_shares_ab2.sql | 22 ++++++++++ .../unibag/tracking_product_shares_ab3.sql | 22 ++++++++++ .../unibag/social_post_views.sql | 20 +++++++++ .../airbyte_tables/unibag/social_posts.sql | 38 ++++++++++++++++ .../unibag/tracking_product_shares.sql | 23 ++++++++++ models/generated/sources.yml | 3 ++ packages.yml | 2 +- profiles.yml | 8 ++-- 22 files changed, 380 insertions(+), 28 deletions(-) create mode 100644 models/generated/airbyte_ctes/unibag/social_post_views_ab1.sql create mode 100644 models/generated/airbyte_ctes/unibag/social_post_views_ab2.sql create mode 100644 models/generated/airbyte_ctes/unibag/social_post_views_ab3.sql create mode 100644 models/generated/airbyte_ctes/unibag/social_posts_ab1.sql create mode 100644 models/generated/airbyte_ctes/unibag/social_posts_ab2.sql create mode 100644 models/generated/airbyte_ctes/unibag/social_posts_ab3.sql create mode 100644 models/generated/airbyte_ctes/unibag/tracking_product_shares_ab1.sql create mode 100644 models/generated/airbyte_ctes/unibag/tracking_product_shares_ab2.sql create mode 100644 models/generated/airbyte_ctes/unibag/tracking_product_shares_ab3.sql create mode 100644 models/generated/airbyte_tables/unibag/social_post_views.sql create mode 100644 models/generated/airbyte_tables/unibag/social_posts.sql create mode 100644 models/generated/airbyte_tables/unibag/tracking_product_shares.sql diff --git a/Makefile b/Makefile index 9dcb83d..01621ed 100644 --- a/Makefile +++ b/Makefile @@ -6,5 +6,5 @@ push: run: dbt deps --profiles-dir=. --project-dir=. #dbt run --profiles-dir=. --project-dir=. --full-refresh - dbt run --profiles-dir=. --project-dir=. --full-refresh --select order_items + dbt run --profiles-dir=. --project-dir=. --full-refresh --select tracking_product_shares diff --git a/dbt_project.yml b/dbt_project.yml index 4869f0a..f957879 100755 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -4,40 +4,40 @@ # Name your package! Package names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: 'airbyte_utils' -version: '1.0' +name: "airbyte_utils" +version: "1.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. Profiles contain # database connection information, and should be configured in the ~/.dbt/profiles.yml file -profile: 'normalize' +profile: "normalize" # These configurations specify where dbt should look for different types of files. -# The `source-paths` config, for example, states that source models can be found +# The `model-paths` config, for example, states that source models can be found # in the "models/" directory. You probably won't need to change these! -source-paths: ["models"] +model-paths: ["models"] docs-paths: ["docs"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] -target-path: "../build" # directory which will store compiled SQL files -log-path: "../logs" # directory which will store DBT logs -modules-path: "../dbt_modules" # directory which will store external DBT dependencies +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +packages-install-path: "../dbt" # directory which will store external DBT dependencies -clean-targets: # directories to be removed by `dbt clean` - - "build" - - "dbt_modules" +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" quoting: database: true -# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) -# all schemas should be unquoted + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted schema: false identifier: true -# You can define configurations for models in the `source-paths` directory here. +# You can define configurations for models in the `model-paths` directory here. # Using these configurations, you can enable or disable models, change how they # are materialized, and more! models: @@ -60,4 +60,4 @@ models: dispatch: - macro_namespace: dbt_utils - search_order: ['airbyte_utils', 'dbt_utils'] + search_order: ["airbyte_utils", "dbt_utils"] diff --git a/destination_catalog.json b/destination_catalog.json index 0d9d850..0118dcb 100644 --- a/destination_catalog.json +++ b/destination_catalog.json @@ -1 +1 @@ -{"streams":[{"stream":{"name":"admin-tags","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"active":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"cash-flows","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"hash":{"type":"string"},"user":{"type":"string"},"value":{"type":"number"},"action":{"type":"string"},"options":{"type":"string"},"category":{"type":"string"},"targetId":{"type":"string"},"createdAt":{"type":"string"},"isAudited":{"type":"boolean"},"updatedAt":{"type":"string"},"newBalance":{"type":"number"},"oldBalance":{"type":"number"},"targetType":{"type":"string"},"isProcessed":{"type":"boolean"},"processStatus":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"cities","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"order":{"type":"number"},"tncId":{"type":"number"},"region":{"type":"string"},"source":{"type":"string"},"tempId":{"type":"number"},"tncCode":{"type":"string"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"customers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"user":{"type":"string"},"email":{"type":"string"},"phone":{"type":"string"},"source":{"type":"string"},"location":{"type":"array"},"createdAt":{"type":"string"},"isDeleted":{"type":"boolean"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"districts","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"city":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"tncId":{"type":"number"},"cityId":{"type":"number"},"source":{"type":"string"},"tncCode":{"type":"string"},"gidoCode":{"type":"number"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"event-rewards","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"cash":{"type":"number"},"type":{"type":"string"},"user":{"type":"string"},"event":{"type":"string"},"status":{"type":"string"},"options":{"type":"string"},"createdAt":{"type":"string"},"updatedat":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"events","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"desc":{"type":"string"},"icon":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"cover":{"type":"string"},"endAt":{"type":"string"},"active":{"type":"boolean"},"notice":{"type":"string"},"screen":{"type":"string"},"article":{"type":"string"},"options":{"type":"string"},"startAt":{"type":"string"},"segments":{"type":"string"},"createdAt":{"type":"string"},"groupType":{"type":"string"},"suppliers":{"type":"array"},"updatedAt":{"type":"string"},"information":{"type":"array"},"displayEndAt":{"type":"string"},"isApplyForAll":{"type":"boolean"},"displayStartAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"inventories","json_schema":{"type":"object","properties":{"id":{"type":"string"},"_id":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"active":{"type":"boolean"},"contact":{"type":"string"},"partner":{"type":"string"},"checksum":{"type":"string"},"location":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"minimumValue":{"type":"number"},"searchString":{"type":"string"},"paymentMethods":{"type":"string"},"canIssueInvoice":{"type":"boolean"},"deliveryMethods":{"type":"array"},"canAutoSendEmail":{"type":"boolean"},"invoiceDeliveryMethod":{"type":"string"},"doesSupportSellyExpress":{"type":"boolean"},"priorityDeliveryServiceCodes":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"color":{"type":"string"},"guide":{"type":"string"},"level":{"type":"number"},"sales":{"type":"number"},"active":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"updatedat":{"type":"string"},"transaction":{"type":"number"},"bonusPercent":{"type":"number"},"bonuspercent":{"type":"number"},"transactionMinValue":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership-histories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"new":{"type":"string"},"old":{"type":"string"},"type":{"type":"string"},"user":{"type":"string"},"createdAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership-orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"status":{"type":"string"},"percent":{"type":"number"},"targetID":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"membershipName":{"type":"string"},"membershipLevel":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-deliveries","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"service":{"type":"string"},"customer":{"type":"string"},"delivery":{"type":"string"},"location":{"type":"string"},"createdAt":{"type":"string"},"promotion":{"type":"string"},"updatedAt":{"type":"string"},"trackingCode":{"type":"string"},"trackingTime":{"type":"string"},"isAdminChange":{"type":"boolean"},"trackingOrderCode":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-histories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"order":{"type":"string"},"status":{"type":"string"},"createdAt":{"type":"string"},"timestamp":{"type":"number"},"updatedAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-items","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"date":{"type":"string"},"user":{"type":"string"},"brand":{"type":"string"},"order":{"type":"string"},"price":{"type":"string"},"status":{"type":"string"},"teamId":{"type":"string"},"product":{"type":"string"},"voucher":{"type":"string"},"customer":{"type":"string"},"quantity":{"type":"number"},"createdAt":{"type":"string"},"inventory":{"type":"string"},"totalSell":{"type":"number"},"updatedAt":{"type":"string"},"cashbackAt":{"type":"string"},"promotions":{"type":"array"},"totalPrice":{"type":"number"},"deliveredAt":{"type":"string"},"teamMemberId":{"type":"string"},"isAssignCoupon":{"type":"boolean"},"totalPromotion":{"type":"number"},"inWholesaleRange":{"type":"boolean"},"voucherCashTotal":{"type":"number"},"wholesaleRangeId":{"type":"string"},"totalWholesaleBonus":{"type":"number"},"totalSellyWholesaleBonus":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"date":{"type":"string"},"hour":{"type":"number"},"note":{"type":"string"},"tags":{"type":"array"},"user":{"type":"string"},"price":{"type":"string"},"banned":{"type":"boolean"},"reason":{"type":"string"},"source":{"type":"string"},"status":{"type":"string"},"teamId":{"type":"string"},"payment":{"type":"string"},"remarks":{"type":"string"},"customer":{"type":"string"},"delivery":{"type":"string"},"isCalled":{"type":"boolean"},"merchant":{"type":"string"},"pickupAt":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"inventory":{"type":"string"},"invoiceId":{"type":"string"},"isDeleted":{"type":"boolean"},"promotion":{"type":"string"},"requestId":{"type":"string"},"restockAt":{"type":"string"},"sendEmail":{"type":"string"},"totalItem":{"type":"number"},"updatedAt":{"type":"string"},"userAgent":{"type":"string"},"wholesale":{"type":"boolean"},"approvedAt":{"type":"string"},"cashbackAt":{"type":"string"},"isPreorder":{"type":"boolean"},"isReviewed":{"type":"boolean"},"rejectedAt":{"type":"string"},"deliveredAt":{"type":"string"},"deliveringAt":{"type":"string"},"deliveryCode":{"type":"string"},"searchString":{"type":"string"},"staffApprove":{"type":"string"},"teamMemberId":{"type":"string"},"processStatus":{"type":"string"},"hookTimeLastAt":{"type":"string"},"isAssignCoupon":{"type":"boolean"},"isAutoApproved":{"type":"boolean"},"outboundRequest":{"type":"string"},"trackingCodeURL":{"type":"string"},"waitingCancelBy":{"type":"string"},"isChangeDelivery":{"type":"boolean"},"estimateCashbackAt":{"type":"string"},"fromNewActiveBuyer":{"type":"boolean"},"isWaitingCancelled":{"type":"boolean"},"fromNewActiveSeller":{"type":"boolean"},"waitingCancelReason":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-categories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"icon":{"type":"string"},"name":{"type":"string"},"color":{"type":"string"},"order":{"type":"number"},"active":{"type":"boolean"},"covers":{"type":"array"},"featured":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"totalProduct":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sku","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"brand":{"type":"string"},"cover":{"type":"string"},"price":{"type":"string"},"active":{"type":"boolean"},"author":{"type":"string"},"source":{"type":"string"},"display":{"type":"boolean"},"picture":{"type":"string"},"product":{"type":"string"},"version":{"type":"number"},"groupSku":{"type":"string"},"unitCode":{"type":"string"},"createdAt":{"type":"string"},"restockAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"couponInfo":{"type":"string"},"properties":{"type":"array"},"canPreorder":{"type":"boolean"},"supplierSku":{"type":"string"},"isOutOfStock":{"type":"boolean"},"pricePercent":{"type":"string"},"searchString":{"type":"string"},"displayInventory":{"type":"boolean"},"showRemainingQuantity":{"type":"boolean"},"quantity_aibyte_transform":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sku-versions","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"price":{"type":"string"},"source":{"type":"string"},"codeSku":{"type":"string"},"picture":{"type":"string"},"product":{"type":"string"},"version":{"type":"number"},"createdAt":{"type":"string"},"properties":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sub-categories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"order":{"type":"number"},"active":{"type":"boolean"},"parent":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"totalProduct":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"products","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"desc":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"temp":{"type":"string"},"brand":{"type":"string"},"order":{"type":"string"},"price":{"type":"string"},"score":{"type":"string"},"active":{"type":"boolean"},"author":{"type":"string"},"guides":{"type":"array"},"photos":{"type":"array"},"videos":{"type":"array"},"quantity":{"type":"number"},"createdAt":{"type":"string"},"documents":{"type":"array"},"shareDesc":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"properties":{"type":"array"},"reActiveAt":{"type":"string"},"inventories":{"type":"array"},"isOutOfStock":{"type":"boolean"},"pricePercent":{"type":"string"},"searchString":{"type":"string"},"subCategories":{"type":"array"},"inactiveReason":{"type":"string"},"propertiesMain":{"type":"string"},"shareStatistic":{"type":"string"},"canIssueInvoice":{"type":"boolean"},"pendingInactive":{"type":"boolean"},"reviewStatistic":{"type":"string"},"pendingInactiveAt":{"type":"string"},"highlightProperties":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"promotion-orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"promotion":{"type":"string"},"updatedAt":{"type":"string"},"cashbackAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"quantityPromotion":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"promotions","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"rest":{"type":"number"},"type":{"type":"string"},"endAt":{"type":"string"},"title":{"type":"string"},"active":{"type":"boolean"},"options":{"type":"string"},"startAt":{"type":"string"},"applyFor":{"type":"string"},"quantity":{"type":"number"},"articleID":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"sharePercent":{"type":"string"},"conditionForUser":{"type":"string"},"isUnlimitedQuantity":{"type":"boolean"},"isApplyForOrderWholesale":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"referrals","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"invitee":{"type":"string"},"createdAt":{"type":"string"},"fromSystem":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"suppliers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"tax":{"type":"string"},"bank":{"type":"string"},"logo":{"type":"string"},"name":{"type":"string"},"active":{"type":"boolean"},"checkSum":{"type":"string"},"checksum":{"type":"string"},"location":{"type":"string"},"contracts":{"type":"array"},"createdAt":{"type":"string"},"documents":{"type":"array"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"inventories":{"type":"array"},"identifyCode":{"type":"number"},"searchString":{"type":"string"},"contractNumber":{"type":"string"},"reviewStatistic":{"type":"string"},"contractSignedAt":{"type":"string"},"emailForInvoices":{"type":"array"},"contractExpiredAt":{"type":"string"},"sendInvoiceRequestEveryDay":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-activities","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"team":{"type":"string"},"action":{"type":"string"},"options":{"type":"string"},"targetId":{"type":"string"},"createdAt":{"type":"string"},"createdBy":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-bonus","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"team":{"type":"string"},"user":{"type":"string"},"status":{"type":"string"},"percent":{"type":"number"},"targetId":{"type":"string"},"teamName":{"type":"string"},"createdAt":{"type":"string"},"teamLevel":{"type":"number"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"teamMember":{"type":"string"},"teamPromotion":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-members","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"role":{"type":"string"},"team":{"type":"string"},"user":{"type":"string"},"isLeft":{"type":"boolean"},"leftAt":{"type":"string"},"joinedAt":{"type":"string"},"roleCode":{"type":"number"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"isRemovedBy":{"type":"string"},"firstOrderAt":{"type":"string"},"searchString":{"type":"string"},"isMemberHomegrown":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"teams","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"desc":{"type":"string"},"logo":{"type":"string"},"name":{"type":"string"},"level":{"type":"string"},"active":{"type":"boolean"},"isFull":{"type":"boolean"},"cityCode":{"type":"number"},"adminUser":{"type":"string"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"inviteLink":{"type":"string"},"information":{"type":"string"},"searchString":{"type":"string"},"inactiveReason":{"type":"string"},"changingNameCount":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"user-identifications","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"dob":{"type":"string"},"name":{"type":"string"},"note":{"type":"string"},"type":{"type":"string"},"user":{"type":"string"},"gender":{"type":"string"},"nation":{"type":"string"},"number":{"type":"string"},"status":{"type":"string"},"address":{"type":"string"},"dobDate":{"type":"string"},"backSide":{"type":"string"},"birthday":{"type":"string"},"createdAt":{"type":"string"},"frontSide":{"type":"string"},"issueDate":{"type":"string"},"updatedAt":{"type":"string"},"expiryDate":{"type":"string"},"issuePlace":{"type":"string"},"timeUpdate":{"type":"number"},"detectInformation":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"users","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"team":{"type":"string"},"zalo":{"type":"string"},"email":{"type":"string"},"phone":{"type":"string"},"avatar":{"type":"string"},"banned":{"type":"boolean"},"segment":{"type":"string"},"facebook":{"type":"string"},"hasOrder":{"type":"boolean"},"referral":{"type":"string"},"segments":{"type":"array"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"membership":{"type":"string"},"lastOrderAt":{"type":"string"},"firstOrderAt":{"type":"string"},"registerFrom":{"type":"string"},"searchString":{"type":"string"},"adjustTracker":{"type":"string"},"isUpdatedInfo":{"type":"boolean"},"identification":{"type":"string"},"lastActivatedAt":{"type":"string"},"lastViewNotificationAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"wards","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"city":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"tncId":{"type":"number"},"cityId":{"type":"number"},"source":{"type":"string"},"tncCode":{"type":"string"},"district":{"type":"string"},"gidoCode":{"type":"number"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"districtId":{"type":"number"},"provinceId":{"type":"number"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]}]} \ No newline at end of file +{"streams":[{"stream":{"name":"admin-tags","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"color":{"type":"string"},"active":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"cash-flows","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"hash":{"type":"string"},"user":{"type":"string"},"value":{"type":"number"},"action":{"type":"string"},"options":{"type":"string"},"category":{"type":"string"},"targetId":{"type":"string"},"createdAt":{"type":"string"},"isAudited":{"type":"boolean"},"updatedAt":{"type":"string"},"newBalance":{"type":"number"},"oldBalance":{"type":"number"},"targetType":{"type":"string"},"isProcessed":{"type":"boolean"},"processStatus":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"cities","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"order":{"type":"number"},"tncId":{"type":"number"},"region":{"type":"string"},"source":{"type":"string"},"tempId":{"type":"number"},"tncCode":{"type":"string"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"customers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"user":{"type":"string"},"phone":{"type":"string"},"source":{"type":"string"},"location":{"type":"array"},"createdAt":{"type":"string"},"isDeleted":{"type":"boolean"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"districts","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"city":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"tncId":{"type":"number"},"cityId":{"type":"number"},"source":{"type":"string"},"tncCode":{"type":"string"},"gidoCode":{"type":"number"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"event-rewards","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"cash":{"type":"number"},"type":{"type":"string"},"user":{"type":"string"},"event":{"type":"string"},"status":{"type":"string"},"options":{"type":"string"},"createdAt":{"type":"string"},"updatedat":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"events","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"desc":{"type":"string"},"icon":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"cover":{"type":"string"},"endAt":{"type":"string"},"active":{"type":"boolean"},"notice":{"type":"string"},"article":{"type":"string"},"options":{"type":"string"},"startAt":{"type":"string"},"segments":{"type":"string"},"createdAt":{"type":"string"},"suppliers":{"type":"array"},"updatedAt":{"type":"string"},"information":{"type":"array"},"displayEndAt":{"type":"string"},"isApplyForAll":{"type":"boolean"},"displayStartAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"inventories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"active":{"type":"boolean"},"contact":{"type":"string"},"partner":{"type":"string"},"checksum":{"type":"string"},"location":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"minimumValue":{"type":"number"},"searchString":{"type":"string"},"paymentMethods":{"type":"string"},"canIssueInvoice":{"type":"boolean"},"deliveryMethods":{"type":"array"},"canAutoSendEmail":{"type":"boolean"},"invoiceDeliveryMethod":{"type":"string"},"doesSupportSellyExpress":{"type":"boolean"},"priorityDeliveryServiceCodes":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"color":{"type":"string"},"guide":{"type":"string"},"level":{"type":"number"},"sales":{"type":"number"},"active":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"transaction":{"type":"number"},"bonusPercent":{"type":"number"},"transactionMinValue":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership-histories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"new":{"type":"string"},"old":{"type":"string"},"type":{"type":"string"},"user":{"type":"string"},"createdAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"membership-orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"status":{"type":"string"},"percent":{"type":"number"},"targetID":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"membershipName":{"type":"string"},"membershipLevel":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-deliveries","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"weight":{"type":"string"},"service":{"type":"string"},"customer":{"type":"string"},"delivery":{"type":"string"},"location":{"type":"string"},"createdAt":{"type":"string"},"promotion":{"type":"string"},"updatedAt":{"type":"string"},"trackingCode":{"type":"string"},"trackingTime":{"type":"string"},"isAdminChange":{"type":"boolean"},"trackingOrderCode":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-histories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"order":{"type":"string"},"status":{"type":"string"},"createdAt":{"type":"string"},"timeStamp":{"type":"number"},"updatedAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"order-items","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"date":{"type":"string"},"user":{"type":"string"},"order":{"type":"string"},"price":{"type":"string"},"status":{"type":"string"},"product":{"type":"string"},"customer":{"type":"string"},"quantity":{"type":"number"},"createdAt":{"type":"string"},"totalSell":{"type":"number"},"updatedAt":{"type":"string"},"cashbackAt":{"type":"string"},"totalPrice":{"type":"number"},"deliveredAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"date":{"type":"string"},"hour":{"type":"number"},"note":{"type":"string"},"skus":{"type":"array"},"tags":{"type":"array"},"user":{"type":"string"},"price":{"type":"string"},"banned":{"type":"boolean"},"reason":{"type":"string"},"source":{"type":"string"},"status":{"type":"string"},"update":{"type":"string"},"payment":{"type":"string"},"remarks":{"type":"string"},"customer":{"type":"string"},"delivery":{"type":"string"},"pickupAt":{"type":"string"},"products":{"type":"array"},"supplier":{"type":"string"},"approveAt":{"type":"string"},"createdAt":{"type":"string"},"inventory":{"type":"string"},"isDeleted":{"type":"boolean"},"promotion":{"type":"string"},"requestId":{"type":"string"},"sendEmail":{"type":"string"},"totalItem":{"type":"number"},"updatedAt":{"type":"string"},"userAgent":{"type":"string"},"wholesale":{"type":"boolean"},"approvedAt":{"type":"string"},"cashbackAt":{"type":"string"},"codeOsiris":{"type":"string"},"isPreorder":{"type":"boolean"},"membership":{"type":"string"},"rejectedAt":{"type":"string"},"deliveredAt":{"type":"string"},"deliveringAt":{"type":"string"},"deliveryCode":{"type":"string"},"searchString":{"type":"string"},"staffApprove":{"type":"string"},"trackingCode":{"type":"string"},"processStatus":{"type":"string"},"hookTimeLastAt":{"type":"string"},"trackingCodeURL":{"type":"string"},"approvePaymentAt":{"type":"string"},"isChangeDelivery":{"type":"boolean"},"trackingOrderCode":{"type":"string"},"estimateCashbackAt":{"type":"string"},"isWaitingCancelled":{"type":"boolean"},"fromNewActiveSeller":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-categories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"icon":{"type":"string"},"name":{"type":"string"},"color":{"type":"string"},"order":{"type":"number"},"active":{"type":"boolean"},"covers":{"type":"array"},"featured":{"type":"boolean"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"totalProduct":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sku","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"type":{"type":"string"},"brand":{"type":"string"},"cover":{"type":"string"},"price":{"type":"string"},"active":{"type":"boolean"},"author":{"type":"string"},"manual":{"type":"boolean"},"source":{"type":"string"},"display":{"type":"boolean"},"picture":{"type":"string"},"product":{"type":"string"},"version":{"type":"number"},"groupSku":{"type":"string"},"unitCode":{"type":"string"},"createdAt":{"type":"string"},"restockAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"properties":{"type":"array"},"canPreorder":{"type":"boolean"},"supplierSku":{"type":"string"},"isOutOfStock":{"type":"boolean"},"pricePercent":{"type":"string"},"searchString":{"type":"string"},"updateReason":{"type":"string"},"updateActiveId":{"type":"string"},"displayInventory":{"type":"boolean"},"showRemainingQuantity":{"type":"boolean"},"quantity_aibyte_transform":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sku-versions","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"sku":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"price":{"type":"string"},"source":{"type":"string"},"codeSku":{"type":"string"},"picture":{"type":"string"},"product":{"type":"string"},"version":{"type":"number"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"properties":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"product-sub-categories","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"name":{"type":"string"},"order":{"type":"number"},"active":{"type":"boolean"},"parent":{"type":"string"},"createdAt":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"totalProduct":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"products","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"desc":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"temp":{"type":"string"},"brand":{"type":"string"},"order":{"type":"string"},"price":{"type":"string"},"score":{"type":"string"},"active":{"type":"boolean"},"author":{"type":"string"},"guides":{"type":"array"},"photos":{"type":"array"},"videos":{"type":"array"},"quantity":{"type":"number"},"createdAt":{"type":"string"},"documents":{"type":"array"},"shareDesc":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"inactiveAt":{"type":"string"},"properties":{"type":"array"},"reActiveAt":{"type":"string"},"inventories":{"type":"array"},"displaySKUId":{"type":"string"},"isOutOfStock":{"type":"boolean"},"pricePercent":{"type":"string"},"searchString":{"type":"string"},"subCategories":{"type":"array"},"inactiveReason":{"type":"string"},"propertiesMain":{"type":"string"},"shareStatistic":{"type":"string"},"updateActiveId":{"type":"string"},"canIssueInvoice":{"type":"boolean"},"pendingInactive":{"type":"boolean"},"reviewStatistic":{"type":"string"},"highlightProperties":{"type":"array"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"promotion-orders","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"order":{"type":"string"},"title":{"type":"string"},"status":{"type":"string"},"supplier":{"type":"string"},"createdAt":{"type":"string"},"promotion":{"type":"string"},"updatedAt":{"type":"string"},"cashbackAt":{"type":"string"},"commission":{"type":"number"},"isRejected":{"type":"boolean"},"orderPriceTotal":{"type":"number"},"quantityPromotion":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"promotions","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"rest":{"type":"number"},"type":{"type":"string"},"endAt":{"type":"string"},"title":{"type":"string"},"active":{"type":"boolean"},"options":{"type":"string"},"startAt":{"type":"string"},"applyFor":{"type":"string"},"quantity":{"type":"number"},"articleID":{"type":"string"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"searchString":{"type":"string"},"conditionForUser":{"type":"string"},"supplierCommission":{"type":"string"},"isUnlimitedQuantity":{"type":"boolean"},"isApplyForOrderWholesale":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"referrals","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"invitee":{"type":"string"},"createdAt":{"type":"string"},"fromSystem":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"social-post-views","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"clientIP":{"type":"string"},"lastViewAt":{"type":"string"},"socialPost":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"social-posts","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"tags":{"type":"array"},"isPin":{"type":"boolean"},"order":{"type":"number"},"title":{"type":"string"},"author":{"type":"string"},"cities":{"type":"array"},"photos":{"type":"array"},"reason":{"type":"string"},"status":{"type":"string"},"videos":{"type":"array"},"content":{"type":"string"},"isTimer":{"type":"boolean"},"startAt":{"type":"string"},"products":{"type":"array"},"createdAt":{"type":"string"},"hasUpdate":{"type":"boolean"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"fromSystem":{"type":"boolean"},"contributor":{"type":"string"},"publishedAt":{"type":"string"},"searchString":{"type":"string"},"updatedCount":{"type":"number"},"shareStatistic":{"type":"string"},"enableNotificationForContributor":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"suppliers","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"tax":{"type":"string"},"bank":{"type":"string"},"logo":{"type":"string"},"name":{"type":"string"},"active":{"type":"boolean"},"balance":{"type":"string"},"checkSum":{"type":"string"},"checksum":{"type":"string"},"location":{"type":"string"},"contracts":{"type":"array"},"createdAt":{"type":"string"},"documents":{"type":"array"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"categories":{"type":"array"},"inventories":{"type":"array"},"identifyCode":{"type":"number"},"searchString":{"type":"string"},"reconcileDays":{"type":"array"},"contractNumber":{"type":"string"},"reviewStatistic":{"type":"string"},"contractSignedAt":{"type":"string"},"emailForInvoices":{"type":"array"},"contractExpiredAt":{"type":"string"},"sendInvoiceRequestEveryDay":{"type":"boolean"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-activities","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"team":{"type":"string"},"action":{"type":"string"},"options":{"type":"string"},"targetId":{"type":"string"},"createdAt":{"type":"string"},"createdBy":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-bonus","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"team":{"type":"string"},"user":{"type":"string"},"status":{"type":"string"},"percent":{"type":"number"},"targetId":{"type":"string"},"teamName":{"type":"string"},"createdAt":{"type":"string"},"teamLevel":{"type":"number"},"updatedAt":{"type":"string"},"commission":{"type":"number"},"teamMember":{"type":"string"},"teamPromotion":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"team-members","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"role":{"type":"string"},"team":{"type":"string"},"user":{"type":"string"},"isLeft":{"type":"boolean"},"leftAt":{"type":"string"},"joinedAt":{"type":"string"},"roleCode":{"type":"number"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"isRemovedBy":{"type":"string"},"searchString":{"type":"string"},"statisticMonth":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"teams","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"desc":{"type":"string"},"logo":{"type":"string"},"name":{"type":"string"},"temp":{"type":"string"},"level":{"type":"string"},"active":{"type":"boolean"},"isFull":{"type":"boolean"},"cityCode":{"type":"number"},"adminUser":{"type":"string"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"inviteLink":{"type":"string"},"information":{"type":"string"},"searchString":{"type":"string"},"inactiveReason":{"type":"string"},"changingNameCount":{"type":"number"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"tracking-product-shares","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"user":{"type":"string"},"source":{"type":"string"},"options":{"type":"string"},"products":{"type":"string"},"createdAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"user-identifications","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"dob":{"type":"string"},"name":{"type":"string"},"note":{"type":"string"},"type":{"type":"string"},"user":{"type":"string"},"gender":{"type":"string"},"nation":{"type":"string"},"number":{"type":"string"},"status":{"type":"string"},"address":{"type":"string"},"backSide":{"type":"string"},"birthday":{"type":"string"},"createdAt":{"type":"string"},"frontSide":{"type":"string"},"issueDate":{"type":"string"},"updatedAt":{"type":"string"},"expiryDate":{"type":"string"},"issuePlace":{"type":"string"},"timeUpdate":{"type":"number"},"detectInformation":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"users","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"code":{"type":"string"},"info":{"type":"string"},"name":{"type":"string"},"team":{"type":"string"},"zalo":{"type":"string"},"email":{"type":"string"},"phone":{"type":"string"},"avatar":{"type":"string"},"banned":{"type":"boolean"},"segment":{"type":"string"},"facebook":{"type":"string"},"hasOrder":{"type":"boolean"},"referral":{"type":"string"},"segments":{"type":"array"},"createdAt":{"type":"string"},"statistic":{"type":"string"},"updatedAt":{"type":"string"},"membership":{"type":"string"},"registerFrom":{"type":"string"},"searchString":{"type":"string"},"isUpdatedInfo":{"type":"boolean"},"identification":{"type":"string"},"lastActivatedAt":{"type":"string"},"lastViewNotificationAt":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]},{"stream":{"name":"wards","json_schema":{"type":"object","properties":{"_id":{"type":"string"},"city":{"type":"string"},"code":{"type":"number"},"name":{"type":"string"},"slug":{"type":"string"},"tncId":{"type":"number"},"cityId":{"type":"number"},"source":{"type":"string"},"tncCode":{"type":"string"},"district":{"type":"string"},"gidoCode":{"type":"number"},"osirisId":{"type":"number"},"updatedAt":{"type":"string"},"districtId":{"type":"number"},"provinceId":{"type":"number"},"searchString":{"type":"string"}}},"supported_sync_modes":["full_refresh","incremental"],"default_cursor_field":[],"source_defined_primary_key":[],"namespace":"unibag"},"sync_mode":"full_refresh","cursor_field":[],"destination_sync_mode":"overwrite","primary_key":[]}]} \ No newline at end of file diff --git a/destination_config.json b/destination_config.json index 97b5347..30d7904 100644 --- a/destination_config.json +++ b/destination_config.json @@ -1 +1 @@ -{"ssl":false,"host":"localhost","port":5555,"schema":"public","database":"selly_etl","password":"123","username":"selly","tunnel_method":{"tunnel_method":"NO_TUNNEL"}} \ No newline at end of file +{"ssl":false,"host":"18.140.112.89","port":5432,"schema":"public","database":"mongo_etl","password":"bvxJaDGW2R55uyDXfODJ2a0Y","username":"selly","tunnel_method":{"tunnel_method":"NO_TUNNEL"}} \ No newline at end of file diff --git a/macros/cross_db_utils/array.sql b/macros/cross_db_utils/array.sql index 4bc642f..9072da2 100644 --- a/macros/cross_db_utils/array.sql +++ b/macros/cross_db_utils/array.sql @@ -5,6 +5,7 @@ - Redshift: -> https://blog.getdbt.com/how-to-unnest-arrays-in-redshift/ - postgres: unnest() -> https://www.postgresqltutorial.com/postgresql-array/ - MSSQL: openjson() –> https://docs.microsoft.com/en-us/sql/relational-databases/json/validate-query-and-change-json-data-with-built-in-functions-sql-server?view=sql-server-ver15 + - ClickHouse: ARRAY JOIN –> https://clickhouse.com/docs/zh/sql-reference/statements/select/array-join/ #} {# cross_join_unnest ------------------------------------------------- #} @@ -21,6 +22,10 @@ cross join unnest({{ array_col }}) as {{ array_col }} {%- endmacro %} +{% macro clickhouse__cross_join_unnest(stream_name, array_col) -%} + ARRAY JOIN {{ array_col }} +{%- endmacro %} + {% macro oracle__cross_join_unnest(stream_name, array_col) -%} {% do exceptions.warn("Normalization does not support unnesting for Oracle yet.") %} {%- endmacro %} diff --git a/macros/cross_db_utils/json_operations.sql b/macros/cross_db_utils/json_operations.sql index 619eaf4..cf52f24 100644 --- a/macros/cross_db_utils/json_operations.sql +++ b/macros/cross_db_utils/json_operations.sql @@ -135,9 +135,9 @@ {% macro clickhouse__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} {%- if from_table|string() == '' %} - JSONExtractRaw({{ json_column }}, {{ format_json_path(json_path_list) }}) + JSONExtractRaw(assumeNotNull({{ json_column }}), {{ format_json_path(json_path_list) }}) {% else %} - JSONExtractRaw({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}) + JSONExtractRaw(assumeNotNull({{ from_table }}.{{ json_column }}), {{ format_json_path(json_path_list) }}) {% endif -%} {%- endmacro %} @@ -180,7 +180,7 @@ {%- endmacro %} {% macro clickhouse__json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} - JSONExtractRaw({{ json_column }}, {{ format_json_path(json_path_list) }}) + JSONExtractRaw(assumeNotNull({{ json_column }}), {{ format_json_path(json_path_list) }}) {%- endmacro %} {# json_extract_array ------------------------------------------------- #} @@ -222,5 +222,5 @@ {%- endmacro %} {% macro clickhouse__json_extract_array(json_column, json_path_list, normalized_json_path) -%} - JSONExtractArrayRaw({{ json_column }}, {{ format_json_path(json_path_list) }}) + JSONExtractArrayRaw(assumeNotNull({{ json_column }}), {{ format_json_path(json_path_list) }}) {%- endmacro %} diff --git a/macros/cross_db_utils/type_conversions.sql b/macros/cross_db_utils/type_conversions.sql index 89dd68b..1ad849b 100644 --- a/macros/cross_db_utils/type_conversions.sql +++ b/macros/cross_db_utils/type_conversions.sql @@ -57,6 +57,11 @@ cast({{ field }} as bit) {%- endmacro %} +{# -- ClickHouse does not support converting string directly to Int8, it must go through int first #} +{% macro clickhouse__cast_to_boolean(field) -%} + IF(lower({{ field }}) = 'true', 1, 0) +{%- endmacro %} + {# empty_string_to_null ------------------------------------------------- #} {% macro empty_string_to_null(field) -%} {{ return(adapter.dispatch('empty_string_to_null')(field)) }} diff --git a/models/generated/airbyte_ctes/unibag/social_post_views_ab1.sql b/models/generated/airbyte_ctes/unibag/social_post_views_ab1.sql new file mode 100644 index 0000000..aa99e8b --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/social_post_views_ab1.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('unibag', '_airbyte_raw_social_post_views') }} +select + {{ json_extract_scalar('_airbyte_data', ['_id'], ['_id']) }} as _id, + {{ json_extract_scalar('_airbyte_data', ['user'], ['user']) }} as {{ adapter.quote('user') }}, + {{ json_extract_scalar('_airbyte_data', ['clientIP'], ['clientIP']) }} as clientip, + {{ json_extract_scalar('_airbyte_data', ['lastViewAt'], ['lastViewAt']) }} as lastviewat, + {{ json_extract_scalar('_airbyte_data', ['socialPost'], ['socialPost']) }} as socialpost, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('unibag', '_airbyte_raw_social_post_views') }} as table_alias +-- social_post_views +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/social_post_views_ab2.sql b/models/generated/airbyte_ctes/unibag/social_post_views_ab2.sql new file mode 100644 index 0000000..8e2cf21 --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/social_post_views_ab2.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('social_post_views_ab1') }} +select + cast(_id as {{ dbt_utils.type_string() }}) as _id, + cast({{ adapter.quote('user') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('user') }}, + cast(clientip as {{ dbt_utils.type_string() }}) as clientip, + cast(lastviewat as {{ dbt_utils.type_string() }}) as lastviewat, + cast(socialpost as {{ dbt_utils.type_string() }}) as socialpost, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('social_post_views_ab1') }} +-- social_post_views +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/social_post_views_ab3.sql b/models/generated/airbyte_ctes/unibag/social_post_views_ab3.sql new file mode 100644 index 0000000..a714392 --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/social_post_views_ab3.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +-- depends_on: {{ ref('social_post_views_ab2') }} +select + {{ dbt_utils.surrogate_key([ + '_id', + adapter.quote('user'), + 'clientip', + 'lastviewat', + 'socialpost', + ]) }} as _airbyte_social_post_views_hashid, + tmp.* +from {{ ref('social_post_views_ab2') }} tmp +-- social_post_views +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/social_posts_ab1.sql b/models/generated/airbyte_ctes/unibag/social_posts_ab1.sql new file mode 100644 index 0000000..9aad28d --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/social_posts_ab1.sql @@ -0,0 +1,43 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('unibag', '_airbyte_raw_social_posts') }} +select + {{ json_extract_scalar('_airbyte_data', ['_id'], ['_id']) }} as _id, + {{ json_extract_array('_airbyte_data', ['tags'], ['tags']) }} as tags, + {{ json_extract_scalar('_airbyte_data', ['isPin'], ['isPin']) }} as ispin, + {{ json_extract_scalar('_airbyte_data', ['order'], ['order']) }} as {{ adapter.quote('order') }}, + {{ json_extract_scalar('_airbyte_data', ['title'], ['title']) }} as title, + {{ json_extract_scalar('_airbyte_data', ['author'], ['author']) }} as author, + {{ json_extract_array('_airbyte_data', ['cities'], ['cities']) }} as cities, + {{ json_extract_array('_airbyte_data', ['photos'], ['photos']) }} as photos, + {{ json_extract_scalar('_airbyte_data', ['reason'], ['reason']) }} as reason, + {{ json_extract_scalar('_airbyte_data', ['status'], ['status']) }} as status, + {{ json_extract_array('_airbyte_data', ['videos'], ['videos']) }} as videos, + {{ json_extract_scalar('_airbyte_data', ['content'], ['content']) }} as {{ adapter.quote('content') }}, + {{ json_extract_scalar('_airbyte_data', ['isTimer'], ['isTimer']) }} as istimer, + {{ json_extract_scalar('_airbyte_data', ['startAt'], ['startAt']) }} as startat, + {{ json_extract_array('_airbyte_data', ['products'], ['products']) }} as products, + {{ json_extract_scalar('_airbyte_data', ['createdAt'], ['createdAt']) }} as createdat, + {{ json_extract_scalar('_airbyte_data', ['hasUpdate'], ['hasUpdate']) }} as hasupdate, + {{ json_extract_scalar('_airbyte_data', ['statistic'], ['statistic']) }} as statistic, + {{ json_extract_scalar('_airbyte_data', ['updatedAt'], ['updatedAt']) }} as updatedat, + {{ json_extract_array('_airbyte_data', ['categories'], ['categories']) }} as categories, + {{ json_extract_scalar('_airbyte_data', ['fromSystem'], ['fromSystem']) }} as fromsystem, + {{ json_extract_scalar('_airbyte_data', ['contributor'], ['contributor']) }} as contributor, + {{ json_extract_scalar('_airbyte_data', ['publishedAt'], ['publishedAt']) }} as publishedat, + {{ json_extract_scalar('_airbyte_data', ['searchString'], ['searchString']) }} as searchstring, + {{ json_extract_scalar('_airbyte_data', ['updatedCount'], ['updatedCount']) }} as updatedcount, + {{ json_extract_scalar('_airbyte_data', ['shareStatistic'], ['shareStatistic']) }} as sharestatistic, + {{ json_extract_scalar('_airbyte_data', ['enableNotificationForContributor'], ['enableNotificationForContributor']) }} as enablenotificationforcontributor, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('unibag', '_airbyte_raw_social_posts') }} as table_alias +-- social_posts +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/social_posts_ab2.sql b/models/generated/airbyte_ctes/unibag/social_posts_ab2.sql new file mode 100644 index 0000000..7300f1b --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/social_posts_ab2.sql @@ -0,0 +1,43 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('social_posts_ab1') }} +select + cast(_id as {{ dbt_utils.type_string() }}) as _id, + tags, + {{ cast_to_boolean('ispin') }} as ispin, + cast({{ adapter.quote('order') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('order') }}, + cast(title as {{ dbt_utils.type_string() }}) as title, + cast(author as {{ dbt_utils.type_string() }}) as author, + cities, + photos, + cast(reason as {{ dbt_utils.type_string() }}) as reason, + cast(status as {{ dbt_utils.type_string() }}) as status, + videos, + cast({{ adapter.quote('content') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('content') }}, + {{ cast_to_boolean('istimer') }} as istimer, + cast(startat as {{ dbt_utils.type_string() }}) as startat, + products, + cast(createdat as {{ dbt_utils.type_string() }}) as createdat, + {{ cast_to_boolean('hasupdate') }} as hasupdate, + cast(statistic as {{ dbt_utils.type_string() }}) as statistic, + cast(updatedat as {{ dbt_utils.type_string() }}) as updatedat, + categories, + {{ cast_to_boolean('fromsystem') }} as fromsystem, + cast(contributor as {{ dbt_utils.type_string() }}) as contributor, + cast(publishedat as {{ dbt_utils.type_string() }}) as publishedat, + cast(searchstring as {{ dbt_utils.type_string() }}) as searchstring, + cast(updatedcount as {{ dbt_utils.type_float() }}) as updatedcount, + cast(sharestatistic as {{ dbt_utils.type_string() }}) as sharestatistic, + {{ cast_to_boolean('enablenotificationforcontributor') }} as enablenotificationforcontributor, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('social_posts_ab1') }} +-- social_posts +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/social_posts_ab3.sql b/models/generated/airbyte_ctes/unibag/social_posts_ab3.sql new file mode 100644 index 0000000..5b932f4 --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/social_posts_ab3.sql @@ -0,0 +1,43 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +-- depends_on: {{ ref('social_posts_ab2') }} +select + {{ dbt_utils.surrogate_key([ + '_id', + array_to_string('tags'), + boolean_to_string('ispin'), + adapter.quote('order'), + 'title', + 'author', + array_to_string('cities'), + array_to_string('photos'), + 'reason', + 'status', + array_to_string('videos'), + adapter.quote('content'), + boolean_to_string('istimer'), + 'startat', + array_to_string('products'), + 'createdat', + boolean_to_string('hasupdate'), + 'statistic', + 'updatedat', + array_to_string('categories'), + boolean_to_string('fromsystem'), + 'contributor', + 'publishedat', + 'searchstring', + 'updatedcount', + 'sharestatistic', + boolean_to_string('enablenotificationforcontributor'), + ]) }} as _airbyte_social_posts_hashid, + tmp.* +from {{ ref('social_posts_ab2') }} tmp +-- social_posts +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab1.sql b/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab1.sql new file mode 100644 index 0000000..963e2ff --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab1.sql @@ -0,0 +1,22 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('unibag', '_airbyte_raw_tracking_product_shares') }} +select + {{ json_extract_scalar('_airbyte_data', ['_id'], ['_id']) }} as _id, + {{ json_extract_scalar('_airbyte_data', ['user'], ['user']) }} as {{ adapter.quote('user') }}, + {{ json_extract_scalar('_airbyte_data', ['source'], ['source']) }} as {{ adapter.quote('source') }}, + {{ json_extract_scalar('_airbyte_data', ['options'], ['options']) }} as {{ adapter.quote('options') }}, + {{ json_extract_scalar('_airbyte_data', ['products'], ['products']) }} as products, + {{ json_extract_scalar('_airbyte_data', ['createdAt'], ['createdAt']) }} as createdat, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('unibag', '_airbyte_raw_tracking_product_shares') }} as table_alias +-- tracking_product_shares +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab2.sql b/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab2.sql new file mode 100644 index 0000000..5faf1fe --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab2.sql @@ -0,0 +1,22 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('tracking_product_shares_ab1') }} +select + cast(_id as {{ dbt_utils.type_string() }}) as _id, + cast({{ adapter.quote('user') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('user') }}, + cast({{ adapter.quote('source') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('source') }}, + cast({{ adapter.quote('options') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('options') }}, + cast(products as {{ dbt_utils.type_string() }}) as products, + cast(createdat as {{ dbt_utils.type_string() }}) as createdat, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('tracking_product_shares_ab1') }} +-- tracking_product_shares +where 1 = 1 + diff --git a/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab3.sql b/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab3.sql new file mode 100644 index 0000000..c3be48b --- /dev/null +++ b/models/generated/airbyte_ctes/unibag/tracking_product_shares_ab3.sql @@ -0,0 +1,22 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_unibag", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +-- depends_on: {{ ref('tracking_product_shares_ab2') }} +select + {{ dbt_utils.surrogate_key([ + '_id', + adapter.quote('user'), + adapter.quote('source'), + adapter.quote('options'), + 'products', + 'createdat', + ]) }} as _airbyte_tracking_product_shares_hashid, + tmp.* +from {{ ref('tracking_product_shares_ab2') }} tmp +-- tracking_product_shares +where 1 = 1 + diff --git a/models/generated/airbyte_tables/unibag/social_post_views.sql b/models/generated/airbyte_tables/unibag/social_post_views.sql new file mode 100644 index 0000000..3810988 --- /dev/null +++ b/models/generated/airbyte_tables/unibag/social_post_views.sql @@ -0,0 +1,20 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "unibag", + tags = [ "top-level" ] +) }} +-- Final base SQL model +-- depends_on: {{ ref('social_post_views_ab3') }} +select + _id, + {{ adapter.quote('user') }} AS seller_id, + socialpost AS social_post_id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_social_post_views_hashid +from {{ ref('social_post_views_ab3') }} +-- social_post_views from {{ source('unibag', '_airbyte_raw_social_post_views') }} +where 1 = 1 + diff --git a/models/generated/airbyte_tables/unibag/social_posts.sql b/models/generated/airbyte_tables/unibag/social_posts.sql new file mode 100644 index 0000000..ee8c4b4 --- /dev/null +++ b/models/generated/airbyte_tables/unibag/social_posts.sql @@ -0,0 +1,38 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "unibag", + tags = [ "top-level" ] +) }} +-- Final base SQL model +-- depends_on: {{ ref('social_posts_ab3') }} +select + _id, + title, + author AS author_id, + {{ adapter.quote('content') }}, + status, + cast({{ adapter.quote('photos') }} AS jsonb) AS photos, + cast({{ adapter.quote('videos') }} AS jsonb) AS videos, + string_to_array(REPLACE(REPLACE(REPLACE(tags::text, '[', ''), ']', ''), '"', ''), ',') AS tags, + COALESCE(cast({{ adapter.quote('statistic') }}::json->>'views' AS integer), 0) AS statistic_views, + COALESCE(cast({{ adapter.quote('statistic') }}::json->>'uniqueViews' AS integer), 0) AS statistic_unique_views, + COALESCE(cast({{ adapter.quote('statistic') }}::json->>'likes' AS integer), 0) AS statistic_likes, + COALESCE(cast({{ adapter.quote('statistic') }}::json->>'shares' AS integer), 0) AS statistic_shares, + COALESCE(cast({{ adapter.quote('statistic') }}::json->>'comments' AS integer), 0) AS statistic_comments, + string_to_array(REPLACE(REPLACE(REPLACE(products::text, '[', ''), ']', ''), '"', ''), ',') AS products, + string_to_array(REPLACE(REPLACE(REPLACE(categories::text, '[', ''), ']', ''), '"', ''), ',') AS categories, + string_to_array(REPLACE(REPLACE(REPLACE(cities::text, '[', ''), ']', ''), '"', ''), ',') AS cities, + {{ adapter.quote('order') }}, + contributor AS contributor_id, + createdat::timestamp AS created_at, + updatedat::timestamp AS updated_at, + publishedat::timestamp AS published_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_social_posts_hashid +from {{ ref('social_posts_ab3') }} +-- social_posts from {{ source('unibag', '_airbyte_raw_social_posts') }} +where 1 = 1 + diff --git a/models/generated/airbyte_tables/unibag/tracking_product_shares.sql b/models/generated/airbyte_tables/unibag/tracking_product_shares.sql new file mode 100644 index 0000000..cfb3ded --- /dev/null +++ b/models/generated/airbyte_tables/unibag/tracking_product_shares.sql @@ -0,0 +1,23 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "unibag", + tags = [ "top-level" ] +) }} +-- Final base SQL model +-- depends_on: {{ ref('tracking_product_shares_ab3') }} +select + _id, + {{ adapter.quote('source') }}, + {{ adapter.quote('user') }} AS seller_id, + products AS product_id, + {{ adapter.quote('options') }}::json->>'action' AS options_action, + createdat::timestamp AS created_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_tracking_product_shares_hashid +from {{ ref('tracking_product_shares_ab3') }} +-- tracking_product_shares from {{ source('unibag', '_airbyte_raw_tracking_product_shares') }} +where 1 = 1 + diff --git a/models/generated/sources.yml b/models/generated/sources.yml index ad32b5f..f424af8 100644 --- a/models/generated/sources.yml +++ b/models/generated/sources.yml @@ -29,11 +29,14 @@ sources: - name: _airbyte_raw_promotion_orders - name: _airbyte_raw_promotions - name: _airbyte_raw_referrals + - name: _airbyte_raw_social_post_views + - name: _airbyte_raw_social_posts - name: _airbyte_raw_suppliers - name: _airbyte_raw_team_activities - name: _airbyte_raw_team_bonus - name: _airbyte_raw_team_members - name: _airbyte_raw_teams + - name: _airbyte_raw_tracking_product_shares - name: _airbyte_raw_user_identifications - name: _airbyte_raw_users - name: _airbyte_raw_wards diff --git a/packages.yml b/packages.yml index 4b74445..33b4edd 100755 --- a/packages.yml +++ b/packages.yml @@ -2,4 +2,4 @@ packages: - git: "https://github.com/fishtown-analytics/dbt-utils.git" - revision: 0.7.4 + revision: 0.8.2 diff --git a/profiles.yml b/profiles.yml index 892eaa5..56bc977 100644 --- a/profiles.yml +++ b/profiles.yml @@ -6,10 +6,10 @@ config: normalize: outputs: prod: - dbname: selly_etl - host: localhost - pass: '123' - port: 5555 + dbname: mongo_etl + host: 18.140.112.89 + pass: bvxJaDGW2R55uyDXfODJ2a0Y + port: 5432 schema: public threads: 8 type: postgres