SQL??NoSQL??????????Sqoop
???????????? ???????[ 2016/4/12 13:29:34 ] ????????????? MySQL
????03 ????????
????HDFS?????????????????????????????????????????????????????????С???Mysql?????е?????????????????????????????????????????????????
??????????????????????????????????????HDFS????????????????????????????????????????????????????????????????????????????????????????????????????????????????
??????????????????????????????????????????????????append?????????????????????????????lastmodified??????
?????????????????????????????????
????--check-column
????filed_name
????--incremental
????append|lastmodified
????--last-value
????value
????check_colume?????ü??????????Σ??????id??Σ???????????
????incremental: ??????????????????????????????id???????????????id????????????????????????????????????μ?????????????????????Mysql???棬??????θ??£??????????????????Ρ?????????????????儷???????????????????????1???????????
????last-value: ?????????ε?id?????????ε????
????04 ???hive
????????HDFS?е??????????????????????????????????????й??????????????в????????????????????Hive???????????????
??????????????Hive?н?????????????????????????????Σ????絼????id??name??????Σ???Hive????????????????Ρ?
?????????????external?????????????????????Hive???????????????????????????HDFS???洢????????и????????????Hive?е?????????????????HDFS?У??????????????á?
???????????????ε???????????????????????????滻Mysql???????
????string ==> ?滻char??varchar
????int ==> ?滻int
????float ==> ?滻float
????timestamp ==> ?滻datetime
?????????????????洢????????????????????????????????????磺
????CREATE external TABLE your_table (
????id int??
????name string
????)
????PARTITIONED BY (pdyear string)
????ROW FORMAT DELIMITED fields terminated by '01'
????STORED AS TEXTFILE
????LOCATION '/path/your_table';
????????????????????pdyear????η????’01′???洢??TEXTFILE?????????????????/path/your_table????MySQL????HDFS????????
????????????????????????????????????????
????/path/your_table/pdyear=2015
????/path/your_table/pdyear=2016
???????????????????ж????????????????????hive???У???hive????????????
????alter table your_table add partition (pdyear='2015') location '/path/your_table/pdyear=2015';
????alter table your_table add partition (pdyear='2016') location '/path/your_table/pdyear=2016';
?????????????????????????Hive?н??в????????????????????Hive??hql???????mysql??????????????????????????????????????????
????05 HDFS??????MySQL
??????Hive?н????????е????????????????????????????洢??Mysql?У??????????Hive????У????????????????HDFS?д洢?????????????Sqoop??HDFS?????????MySQL???У??????????á?
???????????????????
export
--connect
jdbc:mysql://1.2.3.4/db_name
--username
your_username
--password
your_passwd
--table
your_table
--input-null-string
'N'
--update-mode
allowinsert
--update-key
id
--export-dir
/path/your_table/
--columns
id??name
--input-fields-terminated-by
'01'
?????????????
????export?????????????
????update-mode??allowinsert????????????????????????Mysql??????????????????и??£??????У???????ж????????update-key????????????????????????????Ρ?
????input-null-string??Hive?У???????NULL????N????????Mysql?У?????MyQL??Null??????????????????????NULL????????’N’
????????????????????Mysql?????????????????????Σ????????????????????????????
????06 ??????
??????MySQL?е????????HDFS?У????HDFS?е???????????Hive??????????????Sqoop???????SQL??NoSQL????????????????????????????NoSQL????????????????????????
???????????????????????漰???????????????????SPASVOС??(021-61079698-8054)?????????????????????????
??????
???????????SQL???SQL??й????е????????????mysql?????????????SQL??????????????mysql????? distinct?÷?MySQL?н???????????з??Mysql????????????????sql server 2000??????????????MySQL Cluster??δ??????????????SQL Server???????????????????λ?????PHP??SQL????????????????????????????SQL??????????SQLд?????SQL Server????????????????????????????????????SQL?????Pythonд???NoSQL?????
???·???
??????????????????
2023/3/23 14:23:39???д?ò??????????
2023/3/22 16:17:39????????????????????Щ??
2022/6/14 16:14:27??????????????????????????
2021/10/18 15:37:44???????????????
2021/9/17 15:19:29???·???????·
2021/9/14 15:42:25?????????????
2021/5/28 17:25:47??????APP??????????
2021/5/8 17:01:11????????
?????????App Bug???????????????????????Jmeter?????????QC??????APP????????????????app?????е????????jenkins+testng+ant+webdriver??????????????JMeter????HTTP???????Selenium 2.0 WebDriver ??????