ODS层数据不做任何处理,完全仿照业务数据库中的表字段,一模一样的创建ODS层对应表。

8张表建表语句:

①用sqoop把导入到HDFS的时候,加了参数–fields-terminated-by “\t”,因此这里ODS层建表的时候也注意相同的分隔符。

②不管是全量导入还是其他形式,都使用分区表,每天保存一次数据。

drop table if exists ods_order_info;
create external table ods_order_info (
    `id` string COMMENT \'订单编号\',
    `total_amount` decimal(10,2) COMMENT \'订单金额\',
    `order_status` string COMMENT \'订单状态\',
    `user_id` string COMMENT \'用户id\',
    `payment_way` string COMMENT \'支付方式\',
    `out_trade_no` string COMMENT \'支付流水号\',
    `create_time` string COMMENT \'创建时间\',
    `operate_time` string COMMENT \'操作时间\'
) COMMENT \'订单表\'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by \'\t\'
location \'/warehouse/gmall/ods/ods_order_info/\'
;
drop table if exists ods_order_detail;
create external table ods_order_detail( 
    `id` string COMMENT \'订单编号\',
    `order_id` string  COMMENT \'订单号\', 
    `user_id` string COMMENT \'用户id\',
    `sku_id` string COMMENT \'商品id\',
    `sku_name` string COMMENT \'商品名称\',
    `order_price` string COMMENT \'商品价格\',
    `sku_num` string COMMENT \'商品数量\',
    `create_time` string COMMENT \'创建时间\'
) COMMENT \'订单明细表\'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by \'\t\' 
location \'/warehouse/gmall/ods/ods_order_detail/\'
;
drop table if exists ods_sku_info;
create external table ods_sku_info( 
    `id` string COMMENT \'skuId\',
    `spu_id` string   COMMENT \'spuid\', 
    `price` decimal(10,2) COMMENT \'价格\',
    `sku_name` string COMMENT \'商品名称\',
    `sku_desc` string COMMENT \'商品描述\',
    `weight` string COMMENT \'重量\',
    `tm_id` string COMMENT \'品牌id\',
    `category3_id` string COMMENT \'品类id\',
    `create_time` string COMMENT \'创建时间\'
) COMMENT \'商品表\'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by \'\t\'
location \'/warehouse/gmall/ods/ods_sku_info/\'
;
drop table if exists ods_user_info;
create external table ods_user_info( 
    `id` string COMMENT \'用户id\',
    `name`  string COMMENT \'姓名\',
    `birthday` string COMMENT \'生日\',
    `gender` string COMMENT \'性别\',
    `email` string COMMENT \'邮箱\',
    `user_level` string COMMENT \'用户等级\',
    `create_time` string COMMENT \'创建时间\'
) COMMENT \'用户信息\'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by \'\t\'
location \'/warehouse/gmall/ods/ods_user_info/\'
;
drop table if exists ods_base_category1;
create external table ods_base_category1( 
    `id` string COMMENT \'id\',
    `name`  string COMMENT \'名称\'
) COMMENT \'商品一级分类\'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by \'\t\'
location \'/warehouse/gmall/ods/ods_base_category1/\'
;
drop table if exists ods_base_category2;
create external table ods_base_category2( 
    `id` string COMMENT \' id\',
    `name` string COMMENT \'名称\',
    category1_id string COMMENT \'一级品类id\'
) COMMENT \'商品二级分类\'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by \'\t\'
location \'/warehouse/gmall/ods/ods_base_category2/\'
;
drop table if exists ods_base_category3;
create external table ods_base_category3(
    `id` string COMMENT \' id\',
    `name`  string COMMENT \'名称\',
    category2_id string COMMENT \'二级品类id\'
) COMMENT \'商品三级分类\'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by \'\t\'
location \'/warehouse/gmall/ods/ods_base_category3/\'
;
drop table if exists ods_payment_info;
create external table ods_payment_info(
    `id`   bigint COMMENT \'编号\',
    `out_trade_no`    string COMMENT \'对外业务编号\',
    `order_id`        string COMMENT \'订单编号\',
    `user_id`         string COMMENT \'用户编号\',
    `alipay_trade_no` string COMMENT \'支付宝交易流水编号\',
    `total_amount`    decimal(16,2) COMMENT \'支付金额\',
    `subject`         string COMMENT \'交易内容\',
    `payment_type`    string COMMENT \'支付类型\',
    `payment_time`    string COMMENT \'支付时间\'
   )  COMMENT \'支付流水表\'
PARTITIONED BY (`dt` string)
row format delimited fields terminated by \'\t\'
location \'/warehouse/gmall/ods/ods_payment_info/\'
;

数据导入脚本:

#!/bin/bash

   APP=gmall
   hive=/opt/module/hive/bin/hive

# 如果是输入的日期按照取输入日期;如果没输入日期取当前时间的前一天
if [ -n "$1" ] ;then
    do_date=$1
else 
    do_date=`date -d "-1 day" +%F`
fi

sql=" 
load data inpath \'/origin_data/$APP/db/order_info/$do_date\' OVERWRITE into table "$APP".ods_order_info partition(dt=\'$do_date\');

load data inpath \'/origin_data/$APP/db/order_detail/$do_date\' OVERWRITE into table "$APP".ods_order_detail partition(dt=\'$do_date\');

load data inpath \'/origin_data/$APP/db/sku_info/$do_date\' OVERWRITE into table "$APP".ods_sku_info partition(dt=\'$do_date\');

load data inpath \'/origin_data/$APP/db/user_info/$do_date\' OVERWRITE into table "$APP".ods_user_info partition(dt=\'$do_date\');

load data inpath \'/origin_data/$APP/db/payment_info/$do_date\' OVERWRITE into table "$APP".ods_payment_info partition(dt=\'$do_date\');

load data inpath \'/origin_data/$APP/db/base_category1/$do_date\' OVERWRITE into table "$APP".ods_base_category1 partition(dt=\'$do_date\');

load data inpath \'/origin_data/$APP/db/base_category2/$do_date\' OVERWRITE into table "$APP".ods_base_category2 partition(dt=\'$do_date\');

load data inpath \'/origin_data/$APP/db/base_category3/$do_date\' OVERWRITE into table "$APP".ods_base_category3 partition(dt=\'$do_date\'); 
"
$hive -e "$sql"

 

版权声明:本文为noyouth原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接和本声明。
本文链接:https://www.cnblogs.com/noyouth/p/13222043.html