????03 ????????
????HDFS?????????????????????????????????????????????????????????С???Mysql?????е?????????????????????????????????????????????????
??????????????????????????????????????HDFS????????????????????????????????????????????????????????????????????????????????????????????????????????????????
??????????????????????????????????????????????????append?????????????????????????????lastmodified??????
?????????????????????????????????
????--check-column
????filed_name
????--incremental
????append|lastmodified
????--last-value
????value
????check_colume?????ü??????????Σ??????id??Σ???????????
????incremental: ??????????????????????????????id???????????????id????????????????????????????????????μ?????????????????????Mysql???棬??????θ??￡??????????????????Ρ?????????????????儷???????????????????????1???????????
????last-value: ?????????ε?id?????????ε????
????04 ???hive
????????HDFS?е??????????????????????????????????????й??????????????в????????????????????Hive???????????????
??????????????Hive?н?????????????????????????????Σ????絼????id??name??????Σ???Hive????????????????Ρ?
?????????????external?????????????????????Hive???????????????????????????HDFS???洢????????и????????????Hive?е?????????????????HDFS?У??????????????á?
???????????????ε???????????????????????????滻Mysql???????
????string ==> ?滻char??varchar
????int ==> ?滻int
????float ==> ?滻float
????timestamp ==> ?滻datetime
?????????????????洢????????????????????????????????????磺
????CREATE external TABLE your_table (
????id int??
????name string
????)
????PARTITIONED BY (pdyear string)
????ROW FORMAT DELIMITED fields terminated by '01'
????STORED AS TEXTFILE
????LOCATION '/path/your_table';
????????????????????pdyear????η????’01′???洢??TEXTFILE?????????????????/path/your_table????MySQL????HDFS????????
????????????????????????????????????????
????/path/your_table/pdyear=2015
????/path/your_table/pdyear=2016
???????????????????ж????????????????????hive???У???hive????????????
????alter table your_table add partition (pdyear='2015') location '/path/your_table/pdyear=2015';
????alter table your_table add partition (pdyear='2016') location '/path/your_table/pdyear=2016';
?????????????????????????Hive?н??в????????????????????Hive??hql???????mysql??????????????????????????????????????????
????05 HDFS??????MySQL
??????Hive?н????????е????????????????????????????洢??Mysql?У??????????Hive????У????????????????HDFS?д洢?????????????Sqoop??HDFS?????????MySQL???У??????????á?
???????????????????
export
--connect
jdbc:mysql://1.2.3.4/db_name
--username
your_username
--password
your_passwd
--table
your_table
--input-null-string
'N'
--update-mode
allowinsert
--update-key
id
--export-dir
/path/your_table/
--columns
id??name
--input-fields-terminated-by
'01'
?????????????
????export?????????????
????update-mode??allowinsert????????????????????????Mysql??????????????????и??￡??????У???????ж????????update-key????????????????????????????Ρ?
????input-null-string??Hive?У???????NULL????N????????Mysql?У?????MyQL??Null??????????????????????NULL????????’N’
????????????????????Mysql?????????????????????Σ????????????????????????????
????06 ??????
??????MySQL?е????????HDFS?У????HDFS?е???????????Hive??????????????Sqoop???????SQL??NoSQL????????????????????????????NoSQL????????????????????????